diff --git a/.gitattributes b/.gitattributes index 6a501bbb417b5f99a5e2e92202706c68d2b55abd..4903fccb8598829a7b70cd486229d466dda8ed04 100644 --- a/.gitattributes +++ b/.gitattributes @@ -45,3 +45,4 @@ ComfyUI/comfy_api/latest/__pycache__/_io.cpython-312.pyc filter=lfs diff=lfs mer ComfyUI/comfy_api_nodes/__pycache__/nodes_bytedance.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text ComfyUI/comfy_api_nodes/__pycache__/nodes_kling.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text ComfyUI/comfy_api_nodes/apis/__pycache__/__init__.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text +ComfyUI/comfy_extras/__pycache__/nodes_gaussian_splat.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text diff --git a/ComfyUI/comfy_execution/graph.py b/ComfyUI/comfy_execution/graph.py new file mode 100644 index 0000000000000000000000000000000000000000..64dec2045e69af64bf2d5a2193a0f7408791b724 --- /dev/null +++ b/ComfyUI/comfy_execution/graph.py @@ -0,0 +1,342 @@ +from typing import Type, Literal + +import nodes +import asyncio +import inspect +from comfy_execution.graph_utils import is_link, ExecutionBlocker +from comfy.comfy_types.node_typing import ComfyNodeABC, InputTypeDict, InputTypeOptions + +# NOTE: ExecutionBlocker code got moved to graph_utils.py to prevent torch being imported too soon during unit tests +ExecutionBlocker = ExecutionBlocker + +class DependencyCycleError(Exception): + pass + +class NodeInputError(Exception): + pass + +class NodeNotFoundError(Exception): + pass + +class DynamicPrompt: + def __init__(self, original_prompt): + # The original prompt provided by the user + self.original_prompt = original_prompt + # Any extra pieces of the graph created during execution + self.ephemeral_prompt = {} + self.ephemeral_parents = {} + self.ephemeral_display = {} + + def get_node(self, node_id): + if node_id in self.ephemeral_prompt: + return self.ephemeral_prompt[node_id] + if node_id in self.original_prompt: + return self.original_prompt[node_id] + raise NodeNotFoundError(f"Node {node_id} not found") + + def has_node(self, node_id): + return node_id in self.original_prompt or node_id in self.ephemeral_prompt + + def add_ephemeral_node(self, node_id, node_info, parent_id, display_id): + self.ephemeral_prompt[node_id] = node_info + self.ephemeral_parents[node_id] = parent_id + self.ephemeral_display[node_id] = display_id + + def get_real_node_id(self, node_id): + while node_id in self.ephemeral_parents: + node_id = self.ephemeral_parents[node_id] + return node_id + + def get_parent_node_id(self, node_id): + return self.ephemeral_parents.get(node_id, None) + + def get_display_node_id(self, node_id): + while node_id in self.ephemeral_display: + node_id = self.ephemeral_display[node_id] + return node_id + + def all_node_ids(self): + return set(self.original_prompt.keys()).union(set(self.ephemeral_prompt.keys())) + + def get_original_prompt(self): + return self.original_prompt + +def get_input_info( + class_def: Type[ComfyNodeABC], + input_name: str, + valid_inputs: InputTypeDict | None = None +) -> tuple[str, Literal["required", "optional", "hidden"], InputTypeOptions] | tuple[None, None, None]: + """Get the input type, category, and extra info for a given input name. + + Arguments: + class_def: The class definition of the node. + input_name: The name of the input to get info for. + valid_inputs: The valid inputs for the node, or None to use the class_def.INPUT_TYPES(). + + Returns: + tuple[str, str, dict] | tuple[None, None, None]: The input type, category, and extra info for the input name. + """ + + valid_inputs = valid_inputs or class_def.INPUT_TYPES() + input_info = None + input_category = None + if "required" in valid_inputs and input_name in valid_inputs["required"]: + input_category = "required" + input_info = valid_inputs["required"][input_name] + elif "optional" in valid_inputs and input_name in valid_inputs["optional"]: + input_category = "optional" + input_info = valid_inputs["optional"][input_name] + elif "hidden" in valid_inputs and input_name in valid_inputs["hidden"]: + input_category = "hidden" + input_info = valid_inputs["hidden"][input_name] + if input_info is None: + return None, None, None + input_type = input_info[0] + if len(input_info) > 1: + extra_info = input_info[1] + else: + extra_info = {} + # if input_type is a list, it is a Combo defined in outdated format; convert it. + # NOTE: uncomment this when we are confident old format going away won't cause too much trouble. + # if isinstance(input_type, list): + # extra_info["options"] = input_type + # input_type = IO.Combo.io_type + return input_type, input_category, extra_info + +class TopologicalSort: + def __init__(self, dynprompt): + self.dynprompt = dynprompt + self.pendingNodes = {} + self.blockCount = {} # Number of nodes this node is directly blocked by + self.blocking = {} # Which nodes are blocked by this node + self.externalBlocks = 0 + self.unblockedEvent = asyncio.Event() + + def get_input_info(self, unique_id, input_name): + class_type = self.dynprompt.get_node(unique_id)["class_type"] + class_def = nodes.NODE_CLASS_MAPPINGS[class_type] + return get_input_info(class_def, input_name) + + def make_input_strong_link(self, to_node_id, to_input): + inputs = self.dynprompt.get_node(to_node_id)["inputs"] + if to_input not in inputs: + raise NodeInputError(f"Node {to_node_id} says it needs input {to_input}, but there is no input to that node at all") + value = inputs[to_input] + if not is_link(value): + raise NodeInputError(f"Node {to_node_id} says it needs input {to_input}, but that value is a constant") + from_node_id, from_socket = value + self.add_strong_link(from_node_id, from_socket, to_node_id) + + def add_strong_link(self, from_node_id, from_socket, to_node_id): + if not self.is_cached(from_node_id): + self.add_node(from_node_id) + if to_node_id not in self.blocking[from_node_id]: + self.blocking[from_node_id][to_node_id] = {} + self.blockCount[to_node_id] += 1 + self.blocking[from_node_id][to_node_id][from_socket] = True + + def add_node(self, node_unique_id, include_lazy=False, subgraph_nodes=None): + node_ids = [node_unique_id] + links = [] + + while len(node_ids) > 0: + unique_id = node_ids.pop() + if unique_id in self.pendingNodes: + continue + + self.pendingNodes[unique_id] = True + self.blockCount[unique_id] = 0 + self.blocking[unique_id] = {} + + inputs = self.dynprompt.get_node(unique_id)["inputs"] + for input_name in inputs: + value = inputs[input_name] + if is_link(value): + from_node_id, from_socket = value + if subgraph_nodes is not None and from_node_id not in subgraph_nodes: + continue + _, _, input_info = self.get_input_info(unique_id, input_name) + is_lazy = input_info is not None and "lazy" in input_info and input_info["lazy"] + if (include_lazy or not is_lazy): + if not self.is_cached(from_node_id): + node_ids.append(from_node_id) + links.append((from_node_id, from_socket, unique_id)) + + for link in links: + self.add_strong_link(*link) + + def add_external_block(self, node_id): + assert node_id in self.blockCount, "Can't add external block to a node that isn't pending" + self.externalBlocks += 1 + self.blockCount[node_id] += 1 + def unblock(): + self.externalBlocks -= 1 + self.blockCount[node_id] -= 1 + self.unblockedEvent.set() + return unblock + + def is_cached(self, node_id): + return False + + def get_ready_nodes(self): + return [node_id for node_id in self.pendingNodes if self.blockCount[node_id] == 0] + + def pop_node(self, unique_id): + del self.pendingNodes[unique_id] + for blocked_node_id in self.blocking[unique_id]: + self.blockCount[blocked_node_id] -= 1 + del self.blocking[unique_id] + + def is_empty(self): + return len(self.pendingNodes) == 0 + +class ExecutionList(TopologicalSort): + """ + ExecutionList implements a topological dissolve of the graph. After a node is staged for execution, + it can still be returned to the graph after having further dependencies added. + """ + def __init__(self, dynprompt, output_cache, output_link_callback=None): + super().__init__(dynprompt) + self.output_cache = output_cache + self.output_link_callback = output_link_callback + self.staged_node_id = None + self.execution_cache = {} + self.execution_cache_listeners = {} + + def is_cached(self, node_id): + return self.output_cache.get_local(node_id) is not None + + def cache_link(self, from_node_id, to_node_id, from_socket=None): + if to_node_id not in self.execution_cache: + self.execution_cache[to_node_id] = {} + value = self.output_cache.get_local(from_node_id) + self.execution_cache[to_node_id][from_node_id] = value + if from_node_id not in self.execution_cache_listeners: + self.execution_cache_listeners[from_node_id] = set() + self.execution_cache_listeners[from_node_id].add((to_node_id, from_socket)) + if value is not None and from_socket is not None and self.output_link_callback is not None: + self.output_link_callback(value.outputs[from_socket]) + + def get_cache(self, from_node_id, to_node_id): + if to_node_id not in self.execution_cache: + return None + value = self.execution_cache[to_node_id].get(from_node_id) + if value is None: + return None + #Write back to the main cache on touch. + self.output_cache.set_local(from_node_id, value) + return value + + def cache_update(self, node_id, value): + if node_id in self.execution_cache_listeners: + for to_node_id, from_socket in self.execution_cache_listeners[node_id]: + if to_node_id in self.execution_cache: + self.execution_cache[to_node_id][node_id] = value + if from_socket is not None and self.output_link_callback is not None: + self.output_link_callback(value.outputs[from_socket]) + + def add_strong_link(self, from_node_id, from_socket, to_node_id): + super().add_strong_link(from_node_id, from_socket, to_node_id) + self.cache_link(from_node_id, to_node_id, from_socket) + + async def stage_node_execution(self): + assert self.staged_node_id is None + if self.is_empty(): + return None, None, None + available = self.get_ready_nodes() + while len(available) == 0 and self.externalBlocks > 0: + # Wait for an external block to be released + await self.unblockedEvent.wait() + self.unblockedEvent.clear() + available = self.get_ready_nodes() + if len(available) == 0: + cycled_nodes = self.get_nodes_in_cycle() + # Because cycles composed entirely of static nodes are caught during initial validation, + # we will 'blame' the first node in the cycle that is not a static node. + blamed_node = cycled_nodes[0] + for node_id in cycled_nodes: + display_node_id = self.dynprompt.get_display_node_id(node_id) + if display_node_id != node_id: + blamed_node = display_node_id + break + ex = DependencyCycleError("Dependency cycle detected") + error_details = { + "node_id": blamed_node, + "exception_message": str(ex), + "exception_type": "graph.DependencyCycleError", + "traceback": [], + "current_inputs": [] + } + return None, error_details, ex + + self.staged_node_id = self.ux_friendly_pick_node(available) + return self.staged_node_id, None, None + + def ux_friendly_pick_node(self, node_list): + # If an output node is available, do that first. + # Technically this has no effect on the overall length of execution, but it feels better as a user + # for a PreviewImage to display a result as soon as it can + # Some other heuristics could probably be used here to improve the UX further. + def is_output(node_id): + class_type = self.dynprompt.get_node(node_id)["class_type"] + class_def = nodes.NODE_CLASS_MAPPINGS[class_type] + if hasattr(class_def, 'OUTPUT_NODE') and class_def.OUTPUT_NODE == True: + return True + return False + + # If an available node is async, do that first. + # This will execute the asynchronous function earlier, reducing the overall time. + def is_async(node_id): + class_type = self.dynprompt.get_node(node_id)["class_type"] + class_def = nodes.NODE_CLASS_MAPPINGS[class_type] + return inspect.iscoroutinefunction(getattr(class_def, class_def.FUNCTION)) + + for node_id in node_list: + if is_output(node_id) or is_async(node_id): + return node_id + + #This should handle the VAEDecode -> preview case + for node_id in node_list: + for blocked_node_id in self.blocking[node_id]: + if is_output(blocked_node_id): + return node_id + + #This should handle the VAELoader -> VAEDecode -> preview case + for node_id in node_list: + for blocked_node_id in self.blocking[node_id]: + for blocked_node_id1 in self.blocking[blocked_node_id]: + if is_output(blocked_node_id1): + return node_id + + #TODO: this function should be improved + return node_list[0] + + def unstage_node_execution(self): + assert self.staged_node_id is not None + self.staged_node_id = None + + def complete_node_execution(self): + node_id = self.staged_node_id + self.pop_node(node_id) + self.execution_cache.pop(node_id, None) + self.execution_cache_listeners.pop(node_id, None) + self.staged_node_id = None + + def get_nodes_in_cycle(self): + # We'll dissolve the graph in reverse topological order to leave only the nodes in the cycle. + # We're skipping some of the performance optimizations from the original TopologicalSort to keep + # the code simple (and because having a cycle in the first place is a catastrophic error) + blocked_by = { node_id: {} for node_id in self.pendingNodes } + for from_node_id in self.blocking: + for to_node_id in self.blocking[from_node_id]: + if True in self.blocking[from_node_id][to_node_id].values(): + blocked_by[to_node_id][from_node_id] = True + to_remove = [node_id for node_id in blocked_by if len(blocked_by[node_id]) == 0] + while len(to_remove) > 0: + for node_id in to_remove: + for to_node_id in blocked_by: + if node_id in blocked_by[to_node_id]: + del blocked_by[to_node_id][node_id] + del blocked_by[node_id] + to_remove = [node_id for node_id in blocked_by if len(blocked_by[node_id]) == 0] + return list(blocked_by.keys()) diff --git a/ComfyUI/comfy_execution/graph_utils.py b/ComfyUI/comfy_execution/graph_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..496d2c634982aa5afc8d58b9db758f97b3b5c2d8 --- /dev/null +++ b/ComfyUI/comfy_execution/graph_utils.py @@ -0,0 +1,155 @@ +def is_link(obj): + if not isinstance(obj, list): + return False + if len(obj) != 2: + return False + if not isinstance(obj[0], str): + return False + if not isinstance(obj[1], int) and not isinstance(obj[1], float): + return False + return True + +# The GraphBuilder is just a utility class that outputs graphs in the form expected by the ComfyUI back-end +class GraphBuilder: + _default_prefix_root = "" + _default_prefix_call_index = 0 + _default_prefix_graph_index = 0 + + def __init__(self, prefix = None): + if prefix is None: + self.prefix = GraphBuilder.alloc_prefix() + else: + self.prefix = prefix + self.nodes = {} + self.id_gen = 1 + + @classmethod + def set_default_prefix(cls, prefix_root, call_index, graph_index = 0): + cls._default_prefix_root = prefix_root + cls._default_prefix_call_index = call_index + cls._default_prefix_graph_index = graph_index + + @classmethod + def alloc_prefix(cls, root=None, call_index=None, graph_index=None): + if root is None: + root = GraphBuilder._default_prefix_root + if call_index is None: + call_index = GraphBuilder._default_prefix_call_index + if graph_index is None: + graph_index = GraphBuilder._default_prefix_graph_index + result = f"{root}.{call_index}.{graph_index}." + GraphBuilder._default_prefix_graph_index += 1 + return result + + def node(self, class_type, id=None, **kwargs): + if id is None: + id = str(self.id_gen) + self.id_gen += 1 + id = self.prefix + id + if id in self.nodes: + return self.nodes[id] + + node = Node(id, class_type, kwargs) + self.nodes[id] = node + return node + + def lookup_node(self, id): + id = self.prefix + id + return self.nodes.get(id) + + def finalize(self): + output = {} + for node_id, node in self.nodes.items(): + output[node_id] = node.serialize() + return output + + def replace_node_output(self, node_id, index, new_value): + node_id = self.prefix + node_id + to_remove = [] + for node in self.nodes.values(): + for key, value in node.inputs.items(): + if is_link(value) and value[0] == node_id and value[1] == index: + if new_value is None: + to_remove.append((node, key)) + else: + node.inputs[key] = new_value + for node, key in to_remove: + del node.inputs[key] + + def remove_node(self, id): + id = self.prefix + id + del self.nodes[id] + +class Node: + def __init__(self, id, class_type, inputs): + self.id = id + self.class_type = class_type + self.inputs = inputs + self.override_display_id = None + + def out(self, index): + return [self.id, index] + + def set_input(self, key, value): + if value is None: + if key in self.inputs: + del self.inputs[key] + else: + self.inputs[key] = value + + def get_input(self, key): + return self.inputs.get(key) + + def set_override_display_id(self, override_display_id): + self.override_display_id = override_display_id + + def serialize(self): + serialized = { + "class_type": self.class_type, + "inputs": self.inputs + } + if self.override_display_id is not None: + serialized["override_display_id"] = self.override_display_id + return serialized + +def add_graph_prefix(graph, outputs, prefix): + # Change the node IDs and any internal links + new_graph = {} + for node_id, node_info in graph.items(): + # Make sure the added nodes have unique IDs + new_node_id = prefix + node_id + new_node = { "class_type": node_info["class_type"], "inputs": {} } + for input_name, input_value in node_info.get("inputs", {}).items(): + if is_link(input_value): + new_node["inputs"][input_name] = [prefix + input_value[0], input_value[1]] + else: + new_node["inputs"][input_name] = input_value + new_graph[new_node_id] = new_node + + # Change the node IDs in the outputs + new_outputs = [] + for n in range(len(outputs)): + output = outputs[n] + if is_link(output): + new_outputs.append([prefix + output[0], output[1]]) + else: + new_outputs.append(output) + + return new_graph, tuple(new_outputs) + +class ExecutionBlocker: + """ + Return this from a node and any users will be blocked with the given error message. + If the message is None, execution will be blocked silently instead. + Generally, you should avoid using this functionality unless absolutely necessary. Whenever it's + possible, a lazy input will be more efficient and have a better user experience. + This functionality is useful in two cases: + 1. You want to conditionally prevent an output node from executing. (Particularly a built-in node + like SaveImage. For your own output nodes, I would recommend just adding a BOOL input and using + lazy evaluation to let it conditionally disable itself.) + 2. You have a node with multiple possible outputs, some of which are invalid and should not be used. + (I would recommend not making nodes like this in the future -- instead, make multiple nodes with + different outputs. Unfortunately, there are several popular existing nodes using this pattern.) + """ + def __init__(self, message): + self.message = message diff --git a/ComfyUI/comfy_execution/jobs.py b/ComfyUI/comfy_execution/jobs.py new file mode 100644 index 0000000000000000000000000000000000000000..34c06363ba30b3edb20b6c36e58bbfb8d874d832 --- /dev/null +++ b/ComfyUI/comfy_execution/jobs.py @@ -0,0 +1,520 @@ +""" +Job utilities for the /api/jobs endpoint. +Provides normalization and helper functions for job status tracking. +""" + +import uuid +from typing import Callable, Optional + +from comfy_api.internal import prune_dict + + +# Result of classifying a job for cancellation. +# 'running' -> job is currently executing (interrupt it) +# 'pending' -> job is queued but not started (dequeue it) +# 'terminal' -> job already finished (present in history); cancel is a no-op +# 'unknown' -> job id is not present anywhere +CANCEL_RUNNING = 'running' +CANCEL_PENDING = 'pending' +CANCEL_TERMINAL = 'terminal' +CANCEL_UNKNOWN = 'unknown' + + +class JobStatus: + """Job status constants.""" + PENDING = 'pending' + IN_PROGRESS = 'in_progress' + COMPLETED = 'completed' + FAILED = 'failed' + CANCELLED = 'cancelled' + + ALL = [PENDING, IN_PROGRESS, COMPLETED, FAILED, CANCELLED] + + +def validate_job_id(value) -> str: + """Validate a client-supplied job (prompt) id. + + Job ids must be UUIDs in the canonical lowercase hyphenated form. The id + is stored and compared verbatim everywhere downstream — history keys, + websocket events, and /interrupt matching — so accepting another spelling + would silently rewrite the client's id and then miss every exact-match + lookup. Rejecting loudly beats that. + + Returns the id unchanged. Raises ValueError when the value is not a + string in canonical UUID form. + """ + if not isinstance(value, str): + raise ValueError(f"job id must be a string, got {type(value).__name__}") + if str(uuid.UUID(value)) != value: + raise ValueError("job id must be a UUID in canonical lowercase hyphenated form") + return value + + +# Media types that can be previewed in the frontend +PREVIEWABLE_MEDIA_TYPES = frozenset({'images', 'video', 'audio', '3d', 'text'}) + +# 3D file extensions for preview fallback (no dedicated media_type exists) +THREE_D_EXTENSIONS = frozenset({'.obj', '.fbx', '.gltf', '.glb', '.usdz'}) + +# Text file extensions for preview fallback (the formats SaveText can produce) +TEXT_EXTENSIONS = frozenset({'.txt', '.md', '.json'}) + + +def has_3d_extension(filename: str) -> bool: + lower = filename.lower() + return any(lower.endswith(ext) for ext in THREE_D_EXTENSIONS) + + +def normalize_output_item(item): + """Normalize a single output list item for the jobs API. + + Returns the normalized item, or None to exclude it. + String items with 3D extensions become {filename, type, subfolder} dicts. + """ + if item is None: + return None + if isinstance(item, str): + if has_3d_extension(item): + return {'filename': item, 'type': 'output', 'subfolder': '', 'mediaType': '3d'} + return None + if isinstance(item, dict): + return item + return None + + +def normalize_outputs(outputs: dict) -> dict: + """Normalize raw node outputs for the jobs API. + + Transforms string 3D filenames into file output dicts and removes + None items. All other items (non-3D strings, dicts, etc.) are + preserved as-is. + """ + normalized = {} + for node_id, node_outputs in outputs.items(): + if not isinstance(node_outputs, dict): + normalized[node_id] = node_outputs + continue + normalized_node = {} + for media_type, items in node_outputs.items(): + if media_type == 'animated' or not isinstance(items, list): + normalized_node[media_type] = items + continue + normalized_items = [] + for item in items: + if item is None: + continue + norm = normalize_output_item(item) + normalized_items.append(norm if norm is not None else item) + normalized_node[media_type] = normalized_items + normalized[node_id] = normalized_node + return normalized + +# Text preview truncation limit (1024 characters) to prevent preview_output bloat +TEXT_PREVIEW_MAX_LENGTH = 1024 + + +def _create_text_preview(value: str) -> dict: + """Create a text preview dict with optional truncation. + + Returns: + dict with 'content' and optionally 'truncated' flag + """ + if len(value) <= TEXT_PREVIEW_MAX_LENGTH: + return {'content': value} + return { + 'content': value[:TEXT_PREVIEW_MAX_LENGTH], + 'truncated': True + } + + +def _extract_job_metadata(extra_data: dict) -> tuple[Optional[int], Optional[str]]: + """Extract create_time and workflow_id from extra_data. + + Returns: + tuple: (create_time, workflow_id) + """ + create_time = extra_data.get('create_time') + extra_pnginfo = extra_data.get('extra_pnginfo', {}) + workflow_id = extra_pnginfo.get('workflow', {}).get('id') + return create_time, workflow_id + + +def is_previewable(media_type: str, item: dict) -> bool: + """ + Check if an output item is previewable. + Matches frontend logic in ComfyUI_frontend/src/stores/queueStore.ts + Maintains backwards compatibility with existing logic. + + Priority: + 1. media_type is 'images', 'video', 'audio', '3d', or 'text' + 2. format field starts with 'video/' or 'audio/' + 3. filename has a 3D extension (.obj, .fbx, .gltf, .glb, .usdz) + 4. filename has a text extension (.txt, .md, .json, ...) + """ + if media_type in PREVIEWABLE_MEDIA_TYPES: + return True + + # Check format field (MIME type). + # Maintains backwards compatibility with how custom node outputs are handled in the frontend. + fmt = item.get('format', '') + if fmt and (fmt.startswith('video/') or fmt.startswith('audio/')): + return True + + # Check for 3D and text files by extension + filename = item.get('filename', '').lower() + if any(filename.endswith(ext) for ext in THREE_D_EXTENSIONS): + return True + if any(filename.endswith(ext) for ext in TEXT_EXTENSIONS): + return True + + return False + + +def is_text_preview(media_type: str, item: dict) -> bool: + """ + Check if a previewable output item is textual rather than visual media. + + Saved text files (SaveText's .txt/.md/.json) are real outputs but must not + outrank visual media when picking the job preview. + """ + if media_type == 'text': + return True + filename = item.get('filename', '').lower() + return any(filename.endswith(ext) for ext in TEXT_EXTENSIONS) + + +def normalize_queue_item(item: tuple, status: str) -> dict: + """Convert queue item tuple to unified job dict. + + Expects item with sensitive data already removed (5 elements). + """ + priority, prompt_id, _, extra_data, _ = item + create_time, workflow_id = _extract_job_metadata(extra_data) + + return prune_dict({ + 'id': prompt_id, + 'status': status, + 'priority': priority, + 'create_time': create_time, + 'outputs_count': 0, + 'workflow_id': workflow_id, + }) + + +def normalize_history_item(prompt_id: str, history_item: dict, include_outputs: bool = False) -> dict: + """Convert history item dict to unified job dict. + + History items have sensitive data already removed (prompt tuple has 5 elements). + """ + prompt_tuple = history_item['prompt'] + priority, _, prompt, extra_data, _ = prompt_tuple + create_time, workflow_id = _extract_job_metadata(extra_data) + + status_info = history_item.get('status', {}) + status_str = status_info.get('status_str') if status_info else None + + outputs = history_item.get('outputs', {}) + outputs_count, preview_output = get_outputs_summary(outputs) + + execution_error = None + execution_start_time = None + execution_end_time = None + was_interrupted = False + if status_info: + messages = status_info.get('messages', []) + for entry in messages: + if isinstance(entry, (list, tuple)) and len(entry) >= 2: + event_name, event_data = entry[0], entry[1] + if isinstance(event_data, dict): + if event_name == 'execution_start': + execution_start_time = event_data.get('timestamp') + elif event_name in ('execution_success', 'execution_error', 'execution_interrupted'): + execution_end_time = event_data.get('timestamp') + if event_name == 'execution_error': + execution_error = event_data + elif event_name == 'execution_interrupted': + was_interrupted = True + + if status_str == 'success': + status = JobStatus.COMPLETED + elif status_str == 'error': + status = JobStatus.CANCELLED if was_interrupted else JobStatus.FAILED + else: + status = JobStatus.COMPLETED + + job = prune_dict({ + 'id': prompt_id, + 'status': status, + 'priority': priority, + 'create_time': create_time, + 'execution_start_time': execution_start_time, + 'execution_end_time': execution_end_time, + 'execution_error': execution_error, + 'outputs_count': outputs_count, + 'preview_output': preview_output, + 'workflow_id': workflow_id, + }) + + if include_outputs: + job['outputs'] = normalize_outputs(outputs) + job['execution_status'] = status_info + job['workflow'] = { + 'prompt': prompt, + 'extra_data': extra_data, + } + + return job + + +def get_outputs_summary(outputs: dict) -> tuple[int, Optional[dict]]: + """ + Count outputs and find preview in a single pass. + Returns (outputs_count, preview_output). + + Preview priority (matching frontend): + 1. type="output" visual media (saved images/video/audio/3d) + 2. any other previewable visual media (e.g. temp/preview images) + 3. saved text file (e.g. SaveText's .txt/.md/.json) + 4. raw text (only when the job produced nothing else previewable) + + Text is kept in its own slots so node/execution order can't let a text + output mask a visual one (e.g. a text node that runs before an image). + + Text content entries (strings under 'text') are preview-only metadata, + matching the frontend's METADATA_KEYS: they can serve as the fallback + preview but are not counted as outputs. + """ + count = 0 + preview_output = None + fallback_preview = None + text_file_fallback = None + text_fallback = None + + for node_id, node_outputs in outputs.items(): + if not isinstance(node_outputs, dict): + continue + for media_type, items in node_outputs.items(): + # 'animated' is a boolean flag, not actual output items + if media_type == 'animated' or not isinstance(items, list): + continue + + for item in items: + if not isinstance(item, dict): + # Handle text outputs (non-dict items like strings or tuples) + normalized = normalize_output_item(item) + if normalized is None: + # Not a 3D file string — check for text preview + if media_type == 'text': + if preview_output is None: + if isinstance(item, tuple): + text_value = item[0] if item else '' + else: + text_value = str(item) + text_preview = _create_text_preview(text_value) + enriched = { + **text_preview, + 'nodeId': node_id, + 'mediaType': media_type + } + if text_fallback is None: + text_fallback = enriched + continue + # normalize_output_item returned a dict (e.g. 3D file) + item = normalized + + count += 1 + + if preview_output is not None: + continue + + if is_previewable(media_type, item): + enriched = { + **item, + 'nodeId': node_id, + } + if 'mediaType' not in item: + enriched['mediaType'] = media_type + if is_text_preview(media_type, item): + if text_file_fallback is None: + text_file_fallback = enriched + elif item.get('type') == 'output': + preview_output = enriched + elif fallback_preview is None: + fallback_preview = enriched + + return count, preview_output or fallback_preview or text_file_fallback or text_fallback + + +def apply_sorting(jobs: list[dict], sort_by: str, sort_order: str) -> list[dict]: + """Sort jobs list by specified field and order.""" + reverse = (sort_order == 'desc') + + if sort_by == 'execution_duration': + def get_sort_key(job): + start = job.get('execution_start_time', 0) + end = job.get('execution_end_time', 0) + return end - start if end and start else 0 + else: + def get_sort_key(job): + return job.get('create_time', 0) + + return sorted(jobs, key=get_sort_key, reverse=reverse) + + +def get_job(prompt_id: str, running: list, queued: list, history: dict) -> Optional[dict]: + """ + Get a single job by prompt_id from history or queue. + + Args: + prompt_id: The prompt ID to look up + running: List of currently running queue items + queued: List of pending queue items + history: Dict of history items keyed by prompt_id + + Returns: + Job dict with full details, or None if not found + """ + if prompt_id in history: + return normalize_history_item(prompt_id, history[prompt_id], include_outputs=True) + + for item in running: + if item[1] == prompt_id: + return normalize_queue_item(item, JobStatus.IN_PROGRESS) + + for item in queued: + if item[1] == prompt_id: + return normalize_queue_item(item, JobStatus.PENDING) + + return None + + +def get_all_jobs( + running: list, + queued: list, + history: dict, + status_filter: Optional[list[str]] = None, + workflow_id: Optional[str] = None, + sort_by: str = "created_at", + sort_order: str = "desc", + limit: Optional[int] = None, + offset: int = 0 +) -> tuple[list[dict], int]: + """ + Get all jobs (running, pending, completed) with filtering and sorting. + + Args: + running: List of currently running queue items + queued: List of pending queue items + history: Dict of history items keyed by prompt_id + status_filter: List of statuses to include (from JobStatus.ALL) + workflow_id: Filter by workflow ID + sort_by: Field to sort by ('created_at', 'execution_duration') + sort_order: 'asc' or 'desc' + limit: Maximum number of items to return + offset: Number of items to skip + + Returns: + tuple: (jobs_list, total_count) + """ + jobs = [] + + if status_filter is None: + status_filter = JobStatus.ALL + + if JobStatus.IN_PROGRESS in status_filter: + for item in running: + jobs.append(normalize_queue_item(item, JobStatus.IN_PROGRESS)) + + if JobStatus.PENDING in status_filter: + for item in queued: + jobs.append(normalize_queue_item(item, JobStatus.PENDING)) + + history_statuses = {JobStatus.COMPLETED, JobStatus.FAILED, JobStatus.CANCELLED} + requested_history_statuses = history_statuses & set(status_filter) + if requested_history_statuses: + for prompt_id, history_item in history.items(): + job = normalize_history_item(prompt_id, history_item) + if job.get('status') in requested_history_statuses: + jobs.append(job) + + if workflow_id: + jobs = [j for j in jobs if j.get('workflow_id') == workflow_id] + + jobs = apply_sorting(jobs, sort_by, sort_order) + + total_count = len(jobs) + + if offset > 0: + jobs = jobs[offset:] + if limit is not None: + jobs = jobs[:limit] + + return (jobs, total_count) + + +def classify_job_for_cancel(prompt_id: str, running: list, queued: list, history: dict) -> str: + """Classify a job id for cancellation. + + Returns one of CANCEL_RUNNING, CANCEL_PENDING, CANCEL_TERMINAL, CANCEL_UNKNOWN. + + Queue items are tuples whose second element (index 1) is the prompt_id. + History is a dict keyed by prompt_id, so a job present there has already + finished and cancelling it is a no-op. + """ + for item in running: + if item[1] == prompt_id: + return CANCEL_RUNNING + for item in queued: + if item[1] == prompt_id: + return CANCEL_PENDING + if prompt_id in history: + return CANCEL_TERMINAL + return CANCEL_UNKNOWN + + +def cancel_job( + prompt_id: str, + running: list, + queued: list, + history: dict, + interrupt: Callable[[str], bool], + dequeue: Callable[[str], bool], +) -> str: + """Cancel a single job by id, regardless of state. + + Maps the cancel onto the runtime's existing mechanics: + - a running job is interrupted via ``interrupt`` + - a pending job is removed from the queue via ``dequeue`` + - a job that already finished (terminal) is a no-op + - an unknown id is a no-op (callers that need fail-fast behaviour should + validate ids up front with ``classify_job_for_cancel``) + + Both ``interrupt`` and ``dequeue`` take the prompt id and return whether + they acted on a job that was *actually* in that state, so the value returned + here reflects what truly happened rather than the (possibly stale) + classification. This matters around the narrow TOCTOU windows where a job + changes state between the caller's snapshot and the action: + + - a job classified RUNNING may have finished before ``interrupt`` fires: + ``interrupt`` returns False and this returns CANCEL_UNKNOWN (no-op). + - a job classified PENDING may have started executing before ``dequeue`` + fires: ``dequeue`` returns False, ``interrupt`` then catches the now- + running job and this returns CANCEL_RUNNING. If it had simply finished + instead, both return False and this returns CANCEL_UNKNOWN. + + ``interrupt`` must be atomic — interrupt the job only if it is still the one + running — so a cancel can never land on an unrelated prompt that started in + the meantime (see ``execution.PromptQueue.interrupt_if_running``). + """ + classification = classify_job_for_cancel(prompt_id, running, queued, history) + if classification == CANCEL_RUNNING: + return CANCEL_RUNNING if interrupt(prompt_id) else CANCEL_UNKNOWN + if classification == CANCEL_PENDING: + if dequeue(prompt_id): + return CANCEL_PENDING + # Left the pending queue between classification and dequeue: if it + # started executing, interrupt the now-running job; otherwise it has + # already finished and the cancel is a genuine no-op. + return CANCEL_RUNNING if interrupt(prompt_id) else CANCEL_UNKNOWN + # CANCEL_TERMINAL and CANCEL_UNKNOWN are intentional no-ops. + return classification diff --git a/ComfyUI/comfy_execution/progress.py b/ComfyUI/comfy_execution/progress.py new file mode 100644 index 0000000000000000000000000000000000000000..731b8dc6630b4877fc2a9d65f7fc42c8e81e0990 --- /dev/null +++ b/ComfyUI/comfy_execution/progress.py @@ -0,0 +1,348 @@ +from typing import TypedDict, Dict, Optional, Tuple +from typing_extensions import override +from PIL import Image +from enum import Enum +from abc import ABC +from tqdm import tqdm +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from comfy_execution.graph import DynamicPrompt +from protocol import BinaryEventTypes +from comfy_api import feature_flags + +PreviewImageTuple = Tuple[str, Image.Image, Optional[int]] + +class NodeState(Enum): + Pending = "pending" + Running = "running" + Finished = "finished" + Error = "error" + + +class NodeProgressState(TypedDict): + """ + A class to represent the state of a node's progress. + """ + + state: NodeState + value: float + max: float + + +class ProgressHandler(ABC): + """ + Abstract base class for progress handlers. + Progress handlers receive progress updates and display them in various ways. + """ + + def __init__(self, name: str): + self.name = name + self.enabled = True + + def set_registry(self, registry: "ProgressRegistry"): + pass + + def start_handler(self, node_id: str, state: NodeProgressState, prompt_id: str): + """Called when a node starts processing""" + pass + + def update_handler( + self, + node_id: str, + value: float, + max_value: float, + state: NodeProgressState, + prompt_id: str, + image: PreviewImageTuple | None = None, + ): + """Called when a node's progress is updated""" + pass + + def finish_handler(self, node_id: str, state: NodeProgressState, prompt_id: str): + """Called when a node finishes processing""" + pass + + def reset(self): + """Called when the progress registry is reset""" + pass + + def enable(self): + """Enable this handler""" + self.enabled = True + + def disable(self): + """Disable this handler""" + self.enabled = False + + +class CLIProgressHandler(ProgressHandler): + """ + Handler that displays progress using tqdm progress bars in the CLI. + """ + + def __init__(self): + super().__init__("cli") + self.progress_bars: Dict[str, tqdm] = {} + + @override + def start_handler(self, node_id: str, state: NodeProgressState, prompt_id: str): + # Create a new tqdm progress bar + if node_id not in self.progress_bars: + self.progress_bars[node_id] = tqdm( + total=state["max"], + desc=f"Node {node_id}", + unit="steps", + leave=True, + position=len(self.progress_bars), + ) + + @override + def update_handler( + self, + node_id: str, + value: float, + max_value: float, + state: NodeProgressState, + prompt_id: str, + image: PreviewImageTuple | None = None, + ): + # Handle case where start_handler wasn't called + if node_id not in self.progress_bars: + self.progress_bars[node_id] = tqdm( + total=max_value, + desc=f"Node {node_id}", + unit="steps", + leave=True, + position=len(self.progress_bars), + ) + self.progress_bars[node_id].update(value) + else: + # Update existing progress bar + if max_value != self.progress_bars[node_id].total: + self.progress_bars[node_id].total = max_value + # Calculate the update amount (difference from current position) + current_position = self.progress_bars[node_id].n + update_amount = value - current_position + if update_amount > 0: + self.progress_bars[node_id].update(update_amount) + + @override + def finish_handler(self, node_id: str, state: NodeProgressState, prompt_id: str): + # Complete and close the progress bar if it exists + if node_id in self.progress_bars: + # Ensure the bar shows 100% completion + remaining = state["max"] - self.progress_bars[node_id].n + if remaining > 0: + self.progress_bars[node_id].update(remaining) + self.progress_bars[node_id].close() + del self.progress_bars[node_id] + + @override + def reset(self): + # Close all progress bars + for bar in self.progress_bars.values(): + bar.close() + self.progress_bars.clear() + + +class WebUIProgressHandler(ProgressHandler): + """ + Handler that sends progress updates to the WebUI via WebSockets. + """ + + def __init__(self, server_instance): + super().__init__("webui") + self.server_instance = server_instance + + def set_registry(self, registry: "ProgressRegistry"): + self.registry = registry + + def _send_progress_state(self, prompt_id: str, nodes: Dict[str, NodeProgressState]): + """Send the current progress state to the client""" + if self.server_instance is None: + return + + # Only send info for non-pending nodes + active_nodes = { + node_id: { + "value": state["value"], + "max": state["max"], + "state": state["state"].value, + "node_id": node_id, + "prompt_id": prompt_id, + "display_node_id": self.registry.dynprompt.get_display_node_id(node_id), + "parent_node_id": self.registry.dynprompt.get_parent_node_id(node_id), + "real_node_id": self.registry.dynprompt.get_real_node_id(node_id), + } + for node_id, state in nodes.items() + if state["state"] != NodeState.Pending + } + + # Send a combined progress_state message with all node states + # Include client_id to ensure message is only sent to the initiating client + self.server_instance.send_sync( + "progress_state", {"prompt_id": prompt_id, "nodes": active_nodes}, self.server_instance.client_id + ) + + @override + def start_handler(self, node_id: str, state: NodeProgressState, prompt_id: str): + # Send progress state of all nodes + if self.registry: + self._send_progress_state(prompt_id, self.registry.nodes) + + @override + def update_handler( + self, + node_id: str, + value: float, + max_value: float, + state: NodeProgressState, + prompt_id: str, + image: PreviewImageTuple | None = None, + ): + # Send progress state of all nodes + if self.registry: + self._send_progress_state(prompt_id, self.registry.nodes) + if image: + # Only send new format if client supports it + if feature_flags.supports_feature( + self.server_instance.sockets_metadata, + self.server_instance.client_id, + "supports_preview_metadata", + ): + metadata = { + "node_id": node_id, + "prompt_id": prompt_id, + "display_node_id": self.registry.dynprompt.get_display_node_id( + node_id + ), + "parent_node_id": self.registry.dynprompt.get_parent_node_id( + node_id + ), + "real_node_id": self.registry.dynprompt.get_real_node_id(node_id), + } + self.server_instance.send_sync( + BinaryEventTypes.PREVIEW_IMAGE_WITH_METADATA, + (image, metadata), + self.server_instance.client_id, + ) + + @override + def finish_handler(self, node_id: str, state: NodeProgressState, prompt_id: str): + # Send progress state of all nodes + if self.registry: + self._send_progress_state(prompt_id, self.registry.nodes) + +class ProgressRegistry: + """ + Registry that maintains node progress state and notifies registered handlers. + """ + + def __init__(self, prompt_id: str, dynprompt: "DynamicPrompt"): + self.prompt_id = prompt_id + self.dynprompt = dynprompt + self.nodes: Dict[str, NodeProgressState] = {} + self.handlers: Dict[str, ProgressHandler] = {} + + def register_handler(self, handler: ProgressHandler) -> None: + """Register a progress handler""" + self.handlers[handler.name] = handler + + def unregister_handler(self, handler_name: str) -> None: + """Unregister a progress handler""" + if handler_name in self.handlers: + # Allow handler to clean up resources + self.handlers[handler_name].reset() + del self.handlers[handler_name] + + def enable_handler(self, handler_name: str) -> None: + """Enable a progress handler""" + if handler_name in self.handlers: + self.handlers[handler_name].enable() + + def disable_handler(self, handler_name: str) -> None: + """Disable a progress handler""" + if handler_name in self.handlers: + self.handlers[handler_name].disable() + + def ensure_entry(self, node_id: str) -> NodeProgressState: + """Ensure a node entry exists""" + if node_id not in self.nodes: + self.nodes[node_id] = NodeProgressState( + state=NodeState.Pending, value=0, max=1 + ) + return self.nodes[node_id] + + def start_progress(self, node_id: str) -> None: + """Start progress tracking for a node""" + entry = self.ensure_entry(node_id) + entry["state"] = NodeState.Running + entry["value"] = 0.0 + entry["max"] = 1.0 + + # Notify all enabled handlers + for handler in self.handlers.values(): + if handler.enabled: + handler.start_handler(node_id, entry, self.prompt_id) + + def update_progress( + self, node_id: str, value: float, max_value: float, image: PreviewImageTuple | None = None + ) -> None: + """Update progress for a node""" + entry = self.ensure_entry(node_id) + entry["state"] = NodeState.Running + entry["value"] = value + entry["max"] = max_value + + # Notify all enabled handlers + for handler in self.handlers.values(): + if handler.enabled: + handler.update_handler( + node_id, value, max_value, entry, self.prompt_id, image + ) + + def finish_progress(self, node_id: str) -> None: + """Finish progress tracking for a node""" + entry = self.ensure_entry(node_id) + entry["state"] = NodeState.Finished + entry["value"] = entry["max"] + + # Notify all enabled handlers + for handler in self.handlers.values(): + if handler.enabled: + handler.finish_handler(node_id, entry, self.prompt_id) + + def reset_handlers(self) -> None: + """Reset all handlers""" + for handler in self.handlers.values(): + handler.reset() + +# Global registry instance +global_progress_registry: ProgressRegistry | None = None + +def reset_progress_state(prompt_id: str, dynprompt: "DynamicPrompt") -> None: + global global_progress_registry + + # Reset existing handlers if registry exists + if global_progress_registry is not None: + global_progress_registry.reset_handlers() + + # Create new registry + global_progress_registry = ProgressRegistry(prompt_id, dynprompt) + + +def add_progress_handler(handler: ProgressHandler) -> None: + registry = get_progress_state() + handler.set_registry(registry) + registry.register_handler(handler) + + +def get_progress_state() -> ProgressRegistry: + global global_progress_registry + if global_progress_registry is None: + from comfy_execution.graph import DynamicPrompt + + global_progress_registry = ProgressRegistry( + prompt_id="", dynprompt=DynamicPrompt({}) + ) + return global_progress_registry diff --git a/ComfyUI/comfy_execution/utils.py b/ComfyUI/comfy_execution/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..62d32f1013df633889b59d2207154d1cbbbfe347 --- /dev/null +++ b/ComfyUI/comfy_execution/utils.py @@ -0,0 +1,46 @@ +import contextvars +from typing import Optional, NamedTuple + +class ExecutionContext(NamedTuple): + """ + Context information about the currently executing node. + + Attributes: + node_id: The ID of the currently executing node + list_index: The index in a list being processed (for operations on batches/lists) + """ + prompt_id: str + node_id: str + list_index: Optional[int] + +current_executing_context: contextvars.ContextVar[Optional[ExecutionContext]] = contextvars.ContextVar("current_executing_context", default=None) + +def get_executing_context() -> Optional[ExecutionContext]: + return current_executing_context.get(None) + +class CurrentNodeContext: + """ + Context manager for setting the current executing node context. + + Sets the current_executing_context on enter and resets it on exit. + + Example: + with CurrentNodeContext(node_id="123", list_index=0): + # Code that should run with the current node context set + process_image() + """ + def __init__(self, prompt_id: str, node_id: str, list_index: Optional[int] = None): + self.context = ExecutionContext( + prompt_id= prompt_id, + node_id= node_id, + list_index= list_index + ) + self.token = None + + def __enter__(self): + self.token = current_executing_context.set(self.context) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.token is not None: + current_executing_context.reset(self.token) diff --git a/ComfyUI/comfy_execution/validation.py b/ComfyUI/comfy_execution/validation.py new file mode 100644 index 0000000000000000000000000000000000000000..ae9a2376ce93b1402774c8c39c131ce092532988 --- /dev/null +++ b/ComfyUI/comfy_execution/validation.py @@ -0,0 +1,58 @@ +from comfy_api.latest import IO + + +def validate_node_input( + received_type: str, input_type: str, strict: bool = False +) -> bool: + """ + received_type and input_type are both strings of the form "T1,T2,...". + + If strict is True, the input_type must contain the received_type. + For example, if received_type is "STRING" and input_type is "STRING,INT", + this will return True. But if received_type is "STRING,INT" and input_type is + "INT", this will return False. + + If strict is False, the input_type must have overlap with the received_type. + For example, if received_type is "STRING,BOOLEAN" and input_type is "STRING,INT", + this will return True. + + Supports pre-union type extension behaviour of ``__ne__`` overrides. + """ + # If the types are exactly the same, we can return immediately + # Use pre-union behaviour: inverse of `__ne__` + # NOTE: this lets legacy '*' Any types work that override the __ne__ method of the str class. + if not received_type != input_type: + return True + + # If one of the types is '*', we can return True immediately; this is the 'Any' type. + if received_type == IO.AnyType.io_type or input_type == IO.AnyType.io_type: + return True + + # If the received type or input_type is a MatchType, we can return True immediately; + # validation for this is handled by the frontend + if received_type == IO.MatchType.io_type or input_type == IO.MatchType.io_type: + return True + + # This accounts for some custom nodes that output lists of options as the type; + # if we ever want to break them on purpose, this can be removed + if isinstance(received_type, list) and input_type == IO.Combo.io_type: + return True + + # Not equal, and not strings + if not isinstance(received_type, str) or not isinstance(input_type, str): + return False + + # Split the type strings into sets for comparison + received_types = set(t.strip() for t in received_type.split(",")) + input_types = set(t.strip() for t in input_type.split(",")) + + # If any of the types is '*', we can return True immediately; this is the 'Any' type. + if IO.AnyType.io_type in received_types or IO.AnyType.io_type in input_types: + return True + + if strict: + # In strict mode, all received types must be in the input types + return received_types.issubset(input_types) + else: + # In non-strict mode, there must be at least one type in common + return len(received_types.intersection(input_types)) > 0 diff --git a/ComfyUI/comfy_extras/__pycache__/color_util.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/color_util.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e5f2198ed4038a041196a1743361caa76790ec1e Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/color_util.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_ace.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_ace.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0a47fd608e7bc1666e90f177cf852b97a96e157e Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_ace.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_advanced_samplers.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_advanced_samplers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aee6e68ed6af4c246d69296dcba79073999afd2f Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_advanced_samplers.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_align_your_steps.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_align_your_steps.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4ec7adedebf1af40971128a6aa40142ab05c38f2 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_align_your_steps.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_apg.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_apg.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e320d2ae580a93e29dd0311abefdcea1770574f1 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_apg.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_ar_video.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_ar_video.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6ca640d73f5a65f56a3b4bb6ec99a8d646db4d40 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_ar_video.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_attention_multiply.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_attention_multiply.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..18cf8eec1d70af106b5d151b195f2245a3e52cb7 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_attention_multiply.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_audio.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_audio.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..53959ec4ad90062b7caa35a6d65f9415bed8dbf1 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_audio.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_audio_encoder.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_audio_encoder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..87cedbb73faeb3904274e5ee79fc0c28a0194af9 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_audio_encoder.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_bernini.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_bernini.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2d1cb53a29fe102c90fa22ad4b99468a893e8873 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_bernini.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_bg_removal.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_bg_removal.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..71b05a34f8ff9962ce293e88a7e8ca14d0581e12 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_bg_removal.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_boogu.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_boogu.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a2ff19c222eb8a408263689ba31ad0ff553e69e6 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_boogu.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_bounding_boxes.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_bounding_boxes.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8914c853a4a2189a684ea209b4b726da626ef053 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_bounding_boxes.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_camera_trajectory.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_camera_trajectory.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f61e39fa3e8fec3557ac080d82020c2614c702e3 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_camera_trajectory.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_canny.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_canny.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..897fd79eb78d28df3c51a368896f9f98bb192673 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_canny.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_cfg.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_cfg.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1fe14d6017504d80fbbc11742ecdd4f9fc3d38ad Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_cfg.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_chroma_radiance.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_chroma_radiance.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4ac4cefa3a0b71477f0117d97567896a0d56a8d6 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_chroma_radiance.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_clip_sdxl.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_clip_sdxl.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a08db9749a5d22c1974fb44241d07f47b63f7600 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_clip_sdxl.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_color.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_color.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0b80b8f08bacc4668928393a165264ed478fe9f8 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_color.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_compositing.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_compositing.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fb7c53908e57a127598e1ad1cd6c16c035fb9120 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_compositing.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_cond.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_cond.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..89f398f9d0cc7a2c3e53215ab27b9c083707a19f Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_cond.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_context_windows.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_context_windows.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..590296509eee698da92e43cc9ec2e9264e01640f Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_context_windows.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_controlnet.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_controlnet.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..db2b2962e0b6ab62f0927df0b1efc290f5163be0 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_controlnet.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_cosmos.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_cosmos.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..82c85028272b9eba9737eecf11ed9366df6eeec3 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_cosmos.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_curve.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_curve.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d2ef004ae03e8536698cb5af126f17f6b1bb52c8 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_curve.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_custom_sampler.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_custom_sampler.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ce0022594149d0ab11294303ab28903813fe3b12 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_custom_sampler.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_dataset.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_dataset.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..beb4b3cd063f015d27e053b43705e5378527e1ec Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_dataset.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_depth_anything_3.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_depth_anything_3.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e43b43bbf57768a3013f4b1d9638adb8fb53457e Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_depth_anything_3.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_differential_diffusion.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_differential_diffusion.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6d3582d9487a9c504ac84cc845d261114538c7cd Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_differential_diffusion.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_easycache.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_easycache.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f806615ce96e30316cb1a3d14475aef58a1eac1e Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_easycache.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_edit_model.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_edit_model.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2ef4f3e570807429a0b1f959f933b5a4b8a1879e Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_edit_model.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_eps.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_eps.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..586fce8254184ef9a7beab623d4e3a09ca3e02ff Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_eps.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_flux.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_flux.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2bb8009552d0e3bd66bf7675596fc58882f708fe Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_flux.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_frame_interpolation.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_frame_interpolation.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aff20f1c05cb452abda973546d6bafdbabde2b00 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_frame_interpolation.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_freelunch.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_freelunch.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..88f6843c606360aef1bc0bafed1b7970bef9e388 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_freelunch.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_fresca.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_fresca.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3c33de297f7af3c43bf6ff77701f684110561919 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_fresca.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_gaussian_splat.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_gaussian_splat.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0396fc51a69750050a267aad13356d55cd656124 --- /dev/null +++ b/ComfyUI/comfy_extras/__pycache__/nodes_gaussian_splat.cpython-312.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:819067e8150f89d64325ffb4548d2b34ba8055fb41dabc9f05afd3f5231abcac +size 132647 diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_gits.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_gits.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7bedf512fbce239cd0a8d95067a1cf1af05d3361 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_gits.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_glsl.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_glsl.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..369c9d0002612bcb2c1bdcce2ea111faa25876ff Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_glsl.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_hidream.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_hidream.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2b30b802b3b515a431fa62123faedbf84dec2cbf Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_hidream.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_hidream_o1.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_hidream_o1.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b222896544c7d7b8d8505ffc659682ba6be27652 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_hidream_o1.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_hooks.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_hooks.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..224842f4d2548342406456f44f583981607ddde0 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_hooks.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_hunyuan.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_hunyuan.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b6b8fb7790a9833771b3bd4a9af2d7905d6aba2f Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_hunyuan.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_hunyuan3d.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_hunyuan3d.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2b06a218c64207eb6f7b3800b55eedb5a4809e3d Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_hunyuan3d.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_hypernetwork.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_hypernetwork.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5d79c50d5646a57875ae9044f95fe463650a61ee Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_hypernetwork.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_hypertile.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_hypertile.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8a05909d8a23c882a31cfb049c18ddef865e716a Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_hypertile.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_ideogram4.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_ideogram4.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ef308425475bf0e063a00a17f9310d323fa52ae5 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_ideogram4.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_image_compare.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_image_compare.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4467416f496678f2a8d2fdbbe4954be357ac2e8a Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_image_compare.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_images.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_images.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bf11d25e2c469c89d16698c667fe981ac27c15ae Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_images.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_ip2p.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_ip2p.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0ac226dd62d7265ef389e40f5debfd99d40b2f1a Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_ip2p.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_joyimage.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_joyimage.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..85a2b2f59d30b12fa1f579de46aad94d7fbbd6d1 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_joyimage.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_json_prompt.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_json_prompt.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f286316a22a81026dd73df8617950e90f127a4ce Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_json_prompt.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_kandinsky5.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_kandinsky5.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..19d9b81c9bc407346a3757a296edb0d57e115b43 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_kandinsky5.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_latent.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_latent.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..eb9e62fb52137dcfad7d5c99b2430e02e950c966 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_latent.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_load_3d.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_load_3d.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..488fc8c5fc036694087d69e5c88632094095d970 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_load_3d.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_logic.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_logic.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1be359a65571b51d0dc948d2ed56c428e07debac Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_logic.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_lora_debug.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_lora_debug.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..92344e61b5cf6ac84587b4a2b69f1ae2d0de6f9d Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_lora_debug.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_lora_extract.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_lora_extract.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..273f931380855a6ae7d9a90cd0f784d2c2266193 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_lora_extract.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_lotus.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_lotus.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4f446704a81af39a49b77065f52d0a7826898bc7 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_lotus.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_lt.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_lt.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..44b7ffaeb435daa782a6a152662c5964f73b4235 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_lt.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_lt_audio.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_lt_audio.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d9884a88021d061b2f9dfc7b50016048045c26f9 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_lt_audio.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_lt_upsampler.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_lt_upsampler.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c226f5df5226dbc1494639ee59c1c11b28e900d6 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_lt_upsampler.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_lumina2.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_lumina2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1b9114e8c31442ad7851428e3e4b9814e692e1ad Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_lumina2.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_mage.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_mage.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..544ce7ae0ce2705ed423aac78e0cd95942002581 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_mage.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_mahiro.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_mahiro.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4a3f70a3821437302bad47ae8911775bcb6328b7 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_mahiro.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_mask.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_mask.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4f210e6600e15958e5d53e2fa1d026f1b5c14ee9 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_mask.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_math.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_math.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..45f820a3ac753c6582d491de90d127b5e523824d Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_math.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_mediapipe.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_mediapipe.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..da63dc3b0eb3a4d427f428318b3efcaa6b05fad0 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_mediapipe.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_minimax_h3.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_minimax_h3.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..744f1bb8d65c2152462c34388246fa7ccb996259 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_minimax_h3.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_mochi.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_mochi.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..79eb271e838733b6d77f2bc1134785656510a547 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_mochi.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_model_advanced.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_model_advanced.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e82dc87806ab7eadfc283fc9a6857c0a681b6ecf Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_model_advanced.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_model_downscale.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_model_downscale.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..07dcf215bc3767e051b3457ca875f028313dccb8 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_model_downscale.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_model_merging.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_model_merging.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5fcb6534e40f8b8d5b4492fd940e761d07f798d4 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_model_merging.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_model_merging_model_specific.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_model_merging_model_specific.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3f99dc5f9c91f7d2c3dfa6915135e142614b709b Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_model_merging_model_specific.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_model_patch.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_model_patch.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1ba989df076662e77ff4a0b2cee31c3eeb78b28a Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_model_patch.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_moge.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_moge.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..da2cc88ae0b8206c1a752bdf0cc86ae9a80f5ad6 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_moge.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_morphology.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_morphology.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ab59c6be1d6a860fe70ad34415322cc87e1a1a5b Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_morphology.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_multigpu.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_multigpu.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..764ab064fd8169a08f0ee901de993af5e00cea2b Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_multigpu.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_nag.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_nag.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f169ccc90998838c16a2447e46e0d572ea7b339e Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_nag.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_nop.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_nop.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..facc04a67a7d74553e44aaab8373e2f00eec00a0 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_nop.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_number_convert.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_number_convert.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5fe49053381760eb70aa7ed140cf83f042f2cfd2 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_number_convert.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_optimalsteps.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_optimalsteps.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e434f025149e3a2c9ba79a38e55a2e1e05ff0a80 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_optimalsteps.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_pag.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_pag.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..936d31e983b94278f4c145e8bfe9fee1d23dc084 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_pag.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_painter.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_painter.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..700f92dcc6968f37b05d2f6e98743b5685129698 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_painter.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_perpneg.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_perpneg.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..660196c7d516836a50def41170df0196eb097115 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_perpneg.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_photomaker.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_photomaker.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..02135315d53df11c28b8897580227edcf052e760 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_photomaker.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_pid.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_pid.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6a6121ef3276a12da4c7009e675bfc298954dca2 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_pid.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_pixart.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_pixart.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..343403656e7f4ad406fe8d8ab4039b797f97ca94 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_pixart.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_post_processing.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_post_processing.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ef9d0b6d0e8a988512033fae182ae676e3abf8a7 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_post_processing.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_preview_any.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_preview_any.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..701dd9d9c0189638d24a8d25374b621c88e10cbb Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_preview_any.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_primitive.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_primitive.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c8e79054219245f749f50e60622e0f894e1698ae Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_primitive.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_qwen.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_qwen.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3a173d23945a3865020600b57ef4fda841c5cafd Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_qwen.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_rebatch.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_rebatch.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..afff0c949088e8e7d801a434b43ccc5d0870446a Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_rebatch.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_replacements.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_replacements.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..816eac488f7041920dc474424f912b991a72d3d6 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_replacements.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_resolution.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_resolution.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6a1c9739ce4a0172f590965dfc00f6ccfeb4fac8 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_resolution.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_rope.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_rope.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..80ee4ce59f76250f9fde8b6587278da3b43df9c3 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_rope.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_rtdetr.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_rtdetr.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..eacde0f1722c4a2cf575d1460c223cf56443e945 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_rtdetr.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_sag.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_sag.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6b42cba1afba4600ce0c3f49005fc8bdf82db075 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_sag.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_sam3.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_sam3.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2e60b1d3e72d12512230a1f7b9b800d975a7ecc4 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_sam3.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_save_3d.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_save_3d.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..64b1cd26e6695285a9fbe93f3f1d18c7e9691004 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_save_3d.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_scail.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_scail.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..201c9d961f63642216ff48782670cee8aac4f6e5 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_scail.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_sd3.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_sd3.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4a8bda688dec0b98c69d9b752a1b219c13608b55 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_sd3.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_sdpose.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_sdpose.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..10475962407a50a3161656bd07bc05b219d625ef Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_sdpose.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_sdupscale.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_sdupscale.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9ae977c0c60f96ea429dfc196551713dd1f85783 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_sdupscale.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_seed.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_seed.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ea75dd4a94b3af8918e793f2146d9422e3de7e83 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_seed.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_seedvr.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_seedvr.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..22f0379240caff0740dbf447df3823f3d6f6f1de Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_seedvr.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_slg.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_slg.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3da764a24d6738a452960e7b51863d584e4f2c02 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_slg.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_stable3d.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_stable3d.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4bc559d4554ca19214bbd05bf8122ec18f8b6f60 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_stable3d.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_stable_cascade.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_stable_cascade.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b63acf88b497b88c241dd3b63a9a11057e7e5fb2 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_stable_cascade.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_string.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_string.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7742765322e6f6c087eb2e54363feec2a0cfb1cd Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_string.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_tcfg.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_tcfg.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2d70af3701815c6ff4f08f57d98f118ccc14640a Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_tcfg.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_text.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_text.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b7ea72303c3a495635cf65b23d96b8aab398cf3b Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_text.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_text_overlay.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_text_overlay.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ab7015144d85fc03b4c66b7839ca7bc65759ee6f Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_text_overlay.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_textgen.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_textgen.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..08ba52cfa7e8a78d1be24e4151b68a92b46f8a2c Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_textgen.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_tomesd.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_tomesd.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5dab8357e298fc6250ea793c048cf8303ee6415e Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_tomesd.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_toolkit.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_toolkit.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4d19c731a2d5bd95a5d00be033d32331557b3297 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_toolkit.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_torch_compile.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_torch_compile.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5ec0bb9e20a1dd414c9686d41f7b7d13afb3f8e1 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_torch_compile.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_train.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_train.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8ffa4b2ce4073c003246a6d4d059f2829ddaef72 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_train.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_triposplat.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_triposplat.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6c9b0767fcbcf09bbdf6090f8637ef246e069005 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_triposplat.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_upscale_model.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_upscale_model.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0835e3a23cde07174b95b0261efd8ac7dbcccfd2 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_upscale_model.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_video.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_video.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..28f67cf4e4bb94fe46dc2e9781a36b2ad64dce4b Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_video.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_video_model.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_video_model.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ec2c4d4f4a85611e38793316c4b298e04e0eb0e4 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_video_model.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_void.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_void.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8d24848f8fe4ac6883a27832f89ed86d1ac8a95d Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_void.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_wan.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_wan.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bac998cfc4f6c3afe17541f392cc1ffe51981ce8 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_wan.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_wandancer.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_wandancer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f1df4c4969edce3f009c4b70cee33df335878e08 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_wandancer.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_wanmove.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_wanmove.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1ab63b8e15a01c890de4e9675729992bdb4291f2 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_wanmove.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_webcam.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_webcam.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..adbdbdfd3ccc2b26cc58d84981270a9cf370b14f Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_webcam.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/nodes_zimage.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/nodes_zimage.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..70d44099d2852ead74aaa45a750c6059a8213b6c Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/nodes_zimage.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/__pycache__/void_noise_warp.cpython-312.pyc b/ComfyUI/comfy_extras/__pycache__/void_noise_warp.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d8116c8b21a75c6f5057637017a71e52c0892b51 Binary files /dev/null and b/ComfyUI/comfy_extras/__pycache__/void_noise_warp.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/chainner_models/__pycache__/model_loading.cpython-312.pyc b/ComfyUI/comfy_extras/chainner_models/__pycache__/model_loading.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..616b3aab73acdc1db2e7c42f469f6b1fcd115f1d Binary files /dev/null and b/ComfyUI/comfy_extras/chainner_models/__pycache__/model_loading.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/chainner_models/model_loading.py b/ComfyUI/comfy_extras/chainner_models/model_loading.py new file mode 100644 index 0000000000000000000000000000000000000000..1bec4476f6171e9f1b2a4a3b967fc63bbf9e9c3c --- /dev/null +++ b/ComfyUI/comfy_extras/chainner_models/model_loading.py @@ -0,0 +1,6 @@ +import logging +from spandrel import ModelLoader + +def load_state_dict(state_dict): + logging.warning("comfy_extras.chainner_models is deprecated and has been replaced by the spandrel library.") + return ModelLoader().load_from_state_dict(state_dict).eval() diff --git a/ComfyUI/comfy_extras/color_util.py b/ComfyUI/comfy_extras/color_util.py new file mode 100644 index 0000000000000000000000000000000000000000..d50795ae352a21467adc906857315363836926da --- /dev/null +++ b/ComfyUI/comfy_extras/color_util.py @@ -0,0 +1,23 @@ +def hex_to_rgb(value: str) -> tuple[int, int, int]: + h = value.lstrip("#") + if len(h) != 6: + return (255, 255, 255) + try: + return (int(h[0:2], 16), int(h[2:4], 16), int(h[4:6], 16)) + except ValueError: + return (255, 255, 255) + + +def readable_color(rgb: tuple[int, int, int]) -> tuple[int, int, int]: + r, g, b = rgb + lum = 0.299 * r + 0.587 * g + 0.114 * b + if lum >= 130: + return (r, g, b) + t = (130 - lum) / (255 - lum) + return (round(r + (255 - r) * t), round(g + (255 - g) * t), round(b + (255 - b) * t)) + + +def normalize_palette(colors) -> list[str]: + if isinstance(colors, dict): + colors = colors.values() + return [c.upper() for c in colors if isinstance(c, str) and c] diff --git a/ComfyUI/comfy_extras/frame_interpolation_models/__pycache__/film_net.cpython-312.pyc b/ComfyUI/comfy_extras/frame_interpolation_models/__pycache__/film_net.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7930a638fdaeae534b6c8004c8b514bc8cec7fa6 Binary files /dev/null and b/ComfyUI/comfy_extras/frame_interpolation_models/__pycache__/film_net.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/frame_interpolation_models/__pycache__/ifnet.cpython-312.pyc b/ComfyUI/comfy_extras/frame_interpolation_models/__pycache__/ifnet.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e6a3a857f77ea901b3d10df44e8d78e5da5dbabd Binary files /dev/null and b/ComfyUI/comfy_extras/frame_interpolation_models/__pycache__/ifnet.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/frame_interpolation_models/film_net.py b/ComfyUI/comfy_extras/frame_interpolation_models/film_net.py new file mode 100644 index 0000000000000000000000000000000000000000..36bc79dc3c0d535875ee48123e8239a75ca016da --- /dev/null +++ b/ComfyUI/comfy_extras/frame_interpolation_models/film_net.py @@ -0,0 +1,261 @@ +"""FILM: Frame Interpolation for Large Motion (ECCV 2022).""" + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import comfy.ops + +ops = comfy.ops.disable_weight_init + + +class FilmConv2d(nn.Module): + """Conv2d with optional LeakyReLU and FILM-style padding.""" + + def __init__(self, in_channels, out_channels, size, activation=True, device=None, dtype=None, operations=ops): + super().__init__() + self.even_pad = not size % 2 + self.conv = operations.Conv2d(in_channels, out_channels, kernel_size=size, padding=size // 2 if size % 2 else 0, device=device, dtype=dtype) + self.activation = nn.LeakyReLU(0.2) if activation else None + + def forward(self, x): + if self.even_pad: + x = F.pad(x, (0, 1, 0, 1)) + x = self.conv(x) + if self.activation is not None: + x = self.activation(x) + return x + + +def _warp_core(image, flow, grid_x, grid_y): + dtype = image.dtype + H, W = flow.shape[2], flow.shape[3] + dx = flow[:, 0].float() / (W * 0.5) + dy = flow[:, 1].float() / (H * 0.5) + grid = torch.stack([grid_x[None, None, :] + dx, grid_y[None, :, None] + dy], dim=3) + return F.grid_sample(image.float(), grid, mode="bilinear", padding_mode="border", align_corners=False).to(dtype) + + +def build_image_pyramid(image, pyramid_levels): + pyramid = [image] + for _ in range(1, pyramid_levels): + image = F.avg_pool2d(image, 2, 2) + pyramid.append(image) + return pyramid + + +def flow_pyramid_synthesis(residual_pyramid): + flow = residual_pyramid[-1] + flow_pyramid = [flow] + for residual_flow in residual_pyramid[:-1][::-1]: + flow = F.interpolate(flow, size=residual_flow.shape[2:4], mode="bilinear", scale_factor=None).mul_(2).add_(residual_flow) + flow_pyramid.append(flow) + flow_pyramid.reverse() + return flow_pyramid + + +def multiply_pyramid(pyramid, scalar): + return [image * scalar[:, None, None, None] for image in pyramid] + + +def pyramid_warp(feature_pyramid, flow_pyramid, warp_fn): + return [warp_fn(features, flow) for features, flow in zip(feature_pyramid, flow_pyramid)] + + +def concatenate_pyramids(pyramid1, pyramid2): + return [torch.cat([f1, f2], dim=1) for f1, f2 in zip(pyramid1, pyramid2)] + + +class SubTreeExtractor(nn.Module): + def __init__(self, in_channels=3, channels=64, n_layers=4, device=None, dtype=None, operations=ops): + super().__init__() + convs = [] + for i in range(n_layers): + out_ch = channels << i + convs.append(nn.Sequential( + FilmConv2d(in_channels, out_ch, 3, device=device, dtype=dtype, operations=operations), + FilmConv2d(out_ch, out_ch, 3, device=device, dtype=dtype, operations=operations))) + in_channels = out_ch + self.convs = nn.ModuleList(convs) + + def forward(self, image, n): + head = image + pyramid = [] + for i, layer in enumerate(self.convs): + head = layer(head) + pyramid.append(head) + if i < n - 1: + head = F.avg_pool2d(head, 2, 2) + return pyramid + + +class FeatureExtractor(nn.Module): + def __init__(self, in_channels=3, channels=64, sub_levels=4, device=None, dtype=None, operations=ops): + super().__init__() + self.extract_sublevels = SubTreeExtractor(in_channels, channels, sub_levels, device=device, dtype=dtype, operations=operations) + self.sub_levels = sub_levels + + def forward(self, image_pyramid): + sub_pyramids = [self.extract_sublevels(image_pyramid[i], min(len(image_pyramid) - i, self.sub_levels)) + for i in range(len(image_pyramid))] + feature_pyramid = [] + for i in range(len(image_pyramid)): + features = sub_pyramids[i][0] + for j in range(1, self.sub_levels): + if j <= i: + features = torch.cat([features, sub_pyramids[i - j][j]], dim=1) + feature_pyramid.append(features) + # Free sub-pyramids no longer needed by future levels + if i >= self.sub_levels - 1: + sub_pyramids[i - self.sub_levels + 1] = None + return feature_pyramid + + +class FlowEstimator(nn.Module): + def __init__(self, in_channels, num_convs, num_filters, device=None, dtype=None, operations=ops): + super().__init__() + self._convs = nn.ModuleList() + for _ in range(num_convs): + self._convs.append(FilmConv2d(in_channels, num_filters, 3, device=device, dtype=dtype, operations=operations)) + in_channels = num_filters + self._convs.append(FilmConv2d(in_channels, num_filters // 2, 1, device=device, dtype=dtype, operations=operations)) + self._convs.append(FilmConv2d(num_filters // 2, 2, 1, activation=False, device=device, dtype=dtype, operations=operations)) + + def forward(self, features_a, features_b): + net = torch.cat([features_a, features_b], dim=1) + for conv in self._convs: + net = conv(net) + return net + + +class PyramidFlowEstimator(nn.Module): + def __init__(self, filters=64, flow_convs=(3, 3, 3, 3), flow_filters=(32, 64, 128, 256), device=None, dtype=None, operations=ops): + super().__init__() + in_channels = filters << 1 + predictors = [] + for i in range(len(flow_convs)): + predictors.append(FlowEstimator(in_channels, flow_convs[i], flow_filters[i], device=device, dtype=dtype, operations=operations)) + in_channels += filters << (i + 2) + self._predictor = predictors[-1] + self._predictors = nn.ModuleList(predictors[:-1][::-1]) + + def forward(self, feature_pyramid_a, feature_pyramid_b, warp_fn): + levels = len(feature_pyramid_a) + v = self._predictor(feature_pyramid_a[-1], feature_pyramid_b[-1]) + residuals = [v] + # Coarse-to-fine: shared predictor for deep levels, then specialized predictors for fine levels + steps = [(i, self._predictor) for i in range(levels - 2, len(self._predictors) - 1, -1)] + steps += [(len(self._predictors) - 1 - k, p) for k, p in enumerate(self._predictors)] + for i, predictor in steps: + v = F.interpolate(v, size=feature_pyramid_a[i].shape[2:4], mode="bilinear").mul_(2) + v_residual = predictor(feature_pyramid_a[i], warp_fn(feature_pyramid_b[i], v)) + residuals.append(v_residual) + v = v.add_(v_residual) + residuals.reverse() + return residuals + + +def _get_fusion_channels(level, filters): + # Per direction: multi-scale features + RGB image (3ch) + flow (2ch), doubled for both directions + return (sum(filters << i for i in range(level)) + 3 + 2) * 2 + + +class Fusion(nn.Module): + def __init__(self, n_layers=4, specialized_layers=3, filters=64, device=None, dtype=None, operations=ops): + super().__init__() + self.output_conv = operations.Conv2d(filters, 3, kernel_size=1, device=device, dtype=dtype) + self.convs = nn.ModuleList() + in_channels = _get_fusion_channels(n_layers, filters) + increase = 0 + for i in range(n_layers)[::-1]: + num_filters = (filters << i) if i < specialized_layers else (filters << specialized_layers) + self.convs.append(nn.ModuleList([ + FilmConv2d(in_channels, num_filters, 2, activation=False, device=device, dtype=dtype, operations=operations), + FilmConv2d(in_channels + (increase or num_filters), num_filters, 3, device=device, dtype=dtype, operations=operations), + FilmConv2d(num_filters, num_filters, 3, device=device, dtype=dtype, operations=operations)])) + in_channels = num_filters + increase = _get_fusion_channels(i, filters) - num_filters // 2 + + def forward(self, pyramid): + net = pyramid[-1] + for k, layers in enumerate(self.convs): + i = len(self.convs) - 1 - k + net = layers[0](F.interpolate(net, size=pyramid[i].shape[2:4], mode="nearest")) + net = layers[2](layers[1](torch.cat([pyramid[i], net], dim=1))) + return self.output_conv(net) + + +class FILMNet(nn.Module): + def __init__(self, pyramid_levels=7, fusion_pyramid_levels=5, specialized_levels=3, sub_levels=4, + filters=64, flow_convs=(3, 3, 3, 3), flow_filters=(32, 64, 128, 256), device=None, dtype=None, operations=ops): + super().__init__() + self.pyramid_levels = pyramid_levels + self.fusion_pyramid_levels = fusion_pyramid_levels + self.extract = FeatureExtractor(3, filters, sub_levels, device=device, dtype=dtype, operations=operations) + self.predict_flow = PyramidFlowEstimator(filters, flow_convs, flow_filters, device=device, dtype=dtype, operations=operations) + self.fuse = Fusion(sub_levels, specialized_levels, filters, device=device, dtype=dtype, operations=operations) + self._warp_grids = {} + + def get_dtype(self): + return self.extract.extract_sublevels.convs[0][0].conv.weight.dtype + + def memory_used_forward(self, shape, dtype): + return 1700 * shape[1] * shape[2] * dtype.itemsize + + def _build_warp_grids(self, H, W, device): + """Pre-compute warp grids for all pyramid levels.""" + if (H, W) in self._warp_grids: + return + self._warp_grids = {} # clear old resolution grids to prevent memory leaks + for _ in range(self.pyramid_levels): + self._warp_grids[(H, W)] = ( + torch.linspace(-(1 - 1 / W), 1 - 1 / W, W, dtype=torch.float32, device=device), + torch.linspace(-(1 - 1 / H), 1 - 1 / H, H, dtype=torch.float32, device=device), + ) + H, W = H // 2, W // 2 + + def warp(self, image, flow): + grid_x, grid_y = self._warp_grids[(flow.shape[2], flow.shape[3])] + return _warp_core(image, flow, grid_x, grid_y) + + def extract_features(self, img): + """Extract image and feature pyramids for a single frame. Can be cached across pairs.""" + image_pyramid = build_image_pyramid(img, self.pyramid_levels) + feature_pyramid = self.extract(image_pyramid) + return image_pyramid, feature_pyramid + + def forward(self, img0, img1, timestep=0.5, cache=None): + # FILM uses a scalar timestep per batch element (spatially-varying timesteps not supported) + t = timestep.mean(dim=(1, 2, 3)).item() if isinstance(timestep, torch.Tensor) else timestep + return self.forward_multi_timestep(img0, img1, [t], cache=cache) + + def forward_multi_timestep(self, img0, img1, timesteps, cache=None): + """Compute flow once, synthesize at multiple timesteps. Expects batch=1 inputs.""" + self._build_warp_grids(img0.shape[2], img0.shape[3], img0.device) + + image_pyr0, feat_pyr0 = cache["img0"] if cache and "img0" in cache else self.extract_features(img0) + image_pyr1, feat_pyr1 = cache["img1"] if cache and "img1" in cache else self.extract_features(img1) + + fwd_flow = flow_pyramid_synthesis(self.predict_flow(feat_pyr0, feat_pyr1, self.warp))[:self.fusion_pyramid_levels] + bwd_flow = flow_pyramid_synthesis(self.predict_flow(feat_pyr1, feat_pyr0, self.warp))[:self.fusion_pyramid_levels] + + # Build warp targets and free full pyramids (only first fpl levels needed from here) + fpl = self.fusion_pyramid_levels + p2w = [concatenate_pyramids(image_pyr0[:fpl], feat_pyr0[:fpl]), + concatenate_pyramids(image_pyr1[:fpl], feat_pyr1[:fpl])] + del image_pyr0, image_pyr1, feat_pyr0, feat_pyr1 + + results = [] + dt_tensors = torch.tensor(timesteps, device=img0.device, dtype=img0.dtype) + for idx in range(len(timesteps)): + batch_dt = dt_tensors[idx:idx + 1] + bwd_scaled = multiply_pyramid(bwd_flow, batch_dt) + fwd_scaled = multiply_pyramid(fwd_flow, 1 - batch_dt) + fwd_warped = pyramid_warp(p2w[0], bwd_scaled, self.warp) + bwd_warped = pyramid_warp(p2w[1], fwd_scaled, self.warp) + aligned = [torch.cat([fw, bw, bf, ff], dim=1) + for fw, bw, bf, ff in zip(fwd_warped, bwd_warped, bwd_scaled, fwd_scaled)] + del fwd_warped, bwd_warped, bwd_scaled, fwd_scaled + results.append(self.fuse(aligned)) + del aligned + return torch.cat(results, dim=0) diff --git a/ComfyUI/comfy_extras/frame_interpolation_models/ifnet.py b/ComfyUI/comfy_extras/frame_interpolation_models/ifnet.py new file mode 100644 index 0000000000000000000000000000000000000000..ad6edbec91af8f84d9539408014758e3f9dd78da --- /dev/null +++ b/ComfyUI/comfy_extras/frame_interpolation_models/ifnet.py @@ -0,0 +1,131 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +import comfy.ops + +ops = comfy.ops.disable_weight_init + + +def _warp(img, flow, warp_grids): + B, _, H, W = img.shape + base_grid, flow_div = warp_grids[(H, W)] + flow_norm = torch.cat([flow[:, 0:1] / flow_div[0], flow[:, 1:2] / flow_div[1]], 1).float() + grid = (base_grid.expand(B, -1, -1, -1) + flow_norm).permute(0, 2, 3, 1) + return F.grid_sample(img.float(), grid, mode="bilinear", padding_mode="border", align_corners=True).to(img.dtype) + + +class Head(nn.Module): + def __init__(self, out_ch=4, device=None, dtype=None, operations=ops): + super().__init__() + self.cnn0 = operations.Conv2d(3, 16, 3, 2, 1, device=device, dtype=dtype) + self.cnn1 = operations.Conv2d(16, 16, 3, 1, 1, device=device, dtype=dtype) + self.cnn2 = operations.Conv2d(16, 16, 3, 1, 1, device=device, dtype=dtype) + self.cnn3 = operations.ConvTranspose2d(16, out_ch, 4, 2, 1, device=device, dtype=dtype) + self.relu = nn.LeakyReLU(0.2, True) + + def forward(self, x): + x = self.relu(self.cnn0(x)) + x = self.relu(self.cnn1(x)) + x = self.relu(self.cnn2(x)) + return self.cnn3(x) + + +class ResConv(nn.Module): + def __init__(self, c, device=None, dtype=None, operations=ops): + super().__init__() + self.conv = operations.Conv2d(c, c, 3, 1, 1, device=device, dtype=dtype) + self.beta = nn.Parameter(torch.ones((1, c, 1, 1), device=device, dtype=dtype)) + self.relu = nn.LeakyReLU(0.2, True) + + def forward(self, x): + return self.relu(torch.addcmul(x, self.conv(x), self.beta)) + + +class IFBlock(nn.Module): + def __init__(self, in_planes, c=64, device=None, dtype=None, operations=ops): + super().__init__() + self.conv0 = nn.Sequential( + nn.Sequential(operations.Conv2d(in_planes, c // 2, 3, 2, 1, device=device, dtype=dtype), nn.LeakyReLU(0.2, True)), + nn.Sequential(operations.Conv2d(c // 2, c, 3, 2, 1, device=device, dtype=dtype), nn.LeakyReLU(0.2, True))) + self.convblock = nn.Sequential(*(ResConv(c, device=device, dtype=dtype, operations=operations) for _ in range(8))) + self.lastconv = nn.Sequential(operations.ConvTranspose2d(c, 4 * 13, 4, 2, 1, device=device, dtype=dtype), nn.PixelShuffle(2)) + + def forward(self, x, flow=None, scale=1): + x = F.interpolate(x, scale_factor=1.0 / scale, mode="bilinear") + if flow is not None: + flow = F.interpolate(flow, scale_factor=1.0 / scale, mode="bilinear").div_(scale) + x = torch.cat((x, flow), 1) + feat = self.convblock(self.conv0(x)) + tmp = F.interpolate(self.lastconv(feat), scale_factor=scale, mode="bilinear") + return tmp[:, :4] * scale, tmp[:, 4:5], tmp[:, 5:] + + +class IFNet(nn.Module): + def __init__(self, head_ch=4, channels=(192, 128, 96, 64, 32), device=None, dtype=None, operations=ops): + super().__init__() + self.encode = Head(out_ch=head_ch, device=device, dtype=dtype, operations=operations) + block_in = [7 + 2 * head_ch] + [8 + 4 + 8 + 2 * head_ch] * 4 + self.blocks = nn.ModuleList([IFBlock(block_in[i], channels[i], device=device, dtype=dtype, operations=operations) for i in range(5)]) + self.scale_list = [16, 8, 4, 2, 1] + self.pad_align = 64 + self._warp_grids = {} + + def get_dtype(self): + return self.encode.cnn0.weight.dtype + + def memory_used_forward(self, shape, dtype): + return 300 * shape[1] * shape[2] * dtype.itemsize + + def _build_warp_grids(self, H, W, device): + if (H, W) in self._warp_grids: + return + self._warp_grids = {} # clear old resolution grids to prevent memory leaks + grid_y, grid_x = torch.meshgrid( + torch.linspace(-1.0, 1.0, H, device=device, dtype=torch.float32), + torch.linspace(-1.0, 1.0, W, device=device, dtype=torch.float32), indexing="ij") + self._warp_grids[(H, W)] = ( + torch.stack((grid_x, grid_y), dim=0).unsqueeze(0), + torch.tensor([(W - 1.0) / 2.0, (H - 1.0) / 2.0], dtype=torch.float32, device=device)) + + def warp(self, img, flow): + return _warp(img, flow, self._warp_grids) + + def extract_features(self, img): + """Extract head features for a single frame. Can be cached across pairs.""" + return self.encode(img) + + def forward(self, img0, img1, timestep=0.5, cache=None): + if not isinstance(timestep, torch.Tensor): + timestep = torch.full((img0.shape[0], 1, img0.shape[2], img0.shape[3]), timestep, device=img0.device, dtype=img0.dtype) + + self._build_warp_grids(img0.shape[2], img0.shape[3], img0.device) + + B = img0.shape[0] + f0 = cache["img0"].expand(B, -1, -1, -1) if cache and "img0" in cache else self.encode(img0) + f1 = cache["img1"].expand(B, -1, -1, -1) if cache and "img1" in cache else self.encode(img1) + flow = mask = feat = None + warped_img0, warped_img1 = img0, img1 + for i, block in enumerate(self.blocks): + if flow is None: + flow, mask, feat = block(torch.cat((img0, img1, f0, f1, timestep), 1), None, scale=self.scale_list[i]) + else: + fd, mask, feat = block( + torch.cat((warped_img0, warped_img1, self.warp(f0, flow[:, :2]), self.warp(f1, flow[:, 2:4]), timestep, mask, feat), 1), + flow, scale=self.scale_list[i]) + flow = flow.add_(fd) + warped_img0 = self.warp(img0, flow[:, :2]) + warped_img1 = self.warp(img1, flow[:, 2:4]) + return torch.lerp(warped_img1, warped_img0, torch.sigmoid(mask)) + + +def detect_rife_config(state_dict): + head_ch = state_dict["encode.cnn3.weight"].shape[1] # ConvTranspose2d: (in_ch, out_ch, kH, kW) + channels = [] + for i in range(5): + key = f"blocks.{i}.conv0.1.0.weight" + if key in state_dict: + channels.append(state_dict[key].shape[0]) + if len(channels) != 5: + raise ValueError(f"Unsupported RIFE model: expected 5 blocks, found {len(channels)}") + return head_ch, channels diff --git a/ComfyUI/comfy_extras/mediapipe/__pycache__/face_geometry.cpython-312.pyc b/ComfyUI/comfy_extras/mediapipe/__pycache__/face_geometry.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..182d58e7abaa88e3f185d06461b4b1de75e4e4bf Binary files /dev/null and b/ComfyUI/comfy_extras/mediapipe/__pycache__/face_geometry.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/mediapipe/__pycache__/face_landmarker.cpython-312.pyc b/ComfyUI/comfy_extras/mediapipe/__pycache__/face_landmarker.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..76aadf1321e3790c636847c4e2a08d53a938b5ad Binary files /dev/null and b/ComfyUI/comfy_extras/mediapipe/__pycache__/face_landmarker.cpython-312.pyc differ diff --git a/ComfyUI/comfy_extras/mediapipe/face_geometry.py b/ComfyUI/comfy_extras/mediapipe/face_geometry.py new file mode 100644 index 0000000000000000000000000000000000000000..4f3813430b4f82a3706c502a0cdf02ac8cba6e9f --- /dev/null +++ b/ComfyUI/comfy_extras/mediapipe/face_geometry.py @@ -0,0 +1,110 @@ +"""Pure-numpy port of MediaPipe's face_geometry (FACE_LANDMARK_PIPELINE mode) ++ weighted Procrustes solver. Computes the 4x4 facial transformation matrix. +""" + + +import math +import numpy as np + + +def _solve_weighted_orthogonal_problem(src: np.ndarray, tgt: np.ndarray, weights: np.ndarray) -> np.ndarray: + """Weighted orthogonal Procrustes (similarity). Returns 4x4 M with + `target ≈ M @ homogeneous(source)` in the weighted LS sense. fp64 for + SVD stability. Port of procrustes_solver.cc.""" + sqrt_w = np.sqrt(weights.astype(np.float64)) + w_total = float((sqrt_w ** 2).sum()) + ws = src.astype(np.float64) * sqrt_w + wt = tgt.astype(np.float64) * sqrt_w + + c_w = (ws @ sqrt_w) / w_total + centered = ws - np.outer(c_w, sqrt_w) + U, _S, Vt = np.linalg.svd(wt @ centered.T, full_matrices=True) + # Disallow reflection: flip the least-significant axis when det(U)·det(V)<0. + post, pre = U.copy(), Vt.T.copy() + if np.linalg.det(post) * np.linalg.det(pre) < 0: + post[:, 2] *= -1.0 + R = post @ pre.T + + denom = float((centered * ws).sum()) + if denom < 1e-12: + raise ValueError("Procrustes denominator collapsed (degenerate source).") + scale = float((R @ centered * wt).sum()) / denom + translation = ((wt - scale * (R @ ws)) @ sqrt_w) / w_total + + M = np.eye(4, dtype=np.float64) + M[:3, :3] = scale * R + M[:3, 3] = translation + return M + + +def _estimate_scale(canonical: np.ndarray, runtime: np.ndarray, weights: np.ndarray) -> float: + """scale = ‖first column of M[:3]‖ per geometry_pipeline.cc::EstimateScale.""" + return float(np.linalg.norm(_solve_weighted_orthogonal_problem(canonical, runtime, weights)[:3, 0])) + + +def solve_facial_transformation_matrix( + landmarks_normalized: np.ndarray, + canonical_vertices: np.ndarray, + procrustes_indices: np.ndarray, + procrustes_weights: np.ndarray, + image_width: int, + image_height: int, + # face_geometry_calculator_options.pbtxt defaults + vertical_fov_degrees: float = 63.0, + near: float = 1.0, +) -> np.ndarray: + """4x4 facial transformation matrix via two-pass scale recovery + `landmarks_normalized` is (N, 3) in MediaPipe normalized convention: x, y + in [0,1] with TOP-LEFT origin, z in width-scaled units. + """ + + h_near = 2.0 * near * math.tan(0.5 * math.radians(vertical_fov_degrees)) + w_near = image_width * h_near / image_height + + sub = procrustes_indices.astype(np.int64) + screen = landmarks_normalized[sub].T.astype(np.float64).copy() + canon = canonical_vertices[sub].T.astype(np.float64).copy() + weights = procrustes_weights.astype(np.float64) + + # ProjectXY (TOP_LEFT y-flip, then scale all 3 axes; z uses x-scale). + screen[1] = 1.0 - screen[1] + screen[0] = screen[0] * w_near - 0.5 * w_near + screen[1] = screen[1] * h_near - 0.5 * h_near + screen[2] = screen[2] * w_near + depth_offset = float(screen[2].mean()) + + def _unproject(s: np.ndarray, scale: float) -> np.ndarray: + s = s.copy() + s[2] = (s[2] - depth_offset + near) / scale + s[0] *= s[2] / near + s[1] *= s[2] / near + s[2] *= -1.0 + return s + + first = screen.copy() + first[2] *= -1.0 + s1 = _estimate_scale(canon, first, weights) # 1st pass: Procrustes on projected XY + s2 = _estimate_scale(canon, _unproject(screen, s1), weights) # 2nd pass: rescale z by s1, un-project XY + return _solve_weighted_orthogonal_problem(canon, _unproject(screen, s1 * s2), weights).astype(np.float32) + + +def transformation_matrix_from_detection(face_dict: dict, image_width: int, image_height: int, canonical_data: dict) -> np.ndarray: + """Adapt a FaceLandmarker face dict to MP's normalized convention and solve. + FaceMesh emits (x, y, z) in 192-canonical units; MP's geometry expects + z_norm = z_canonical * scale_x / image_width""" + + lmks_xy, lmks_3d = face_dict["landmarks_xy"], face_dict["landmarks_3d"] + aug = np.concatenate([lmks_3d[:, :2].astype(np.float64), np.ones((lmks_xy.shape[0], 1))], axis=1) + M, *_ = np.linalg.lstsq(aug, lmks_xy.astype(np.float64), rcond=None) + scale_x = float(np.linalg.norm(M[0])) + z_scale = scale_x / image_width if scale_x > 1e-6 else 1.0 / image_width + + normalized = np.empty((lmks_xy.shape[0], 3), dtype=np.float32) + normalized[:, 0] = lmks_xy[:, 0] / image_width + normalized[:, 1] = lmks_xy[:, 1] / image_height + normalized[:, 2] = lmks_3d[:, 2] * z_scale + return solve_facial_transformation_matrix( + normalized, canonical_data["canonical_vertices"], + canonical_data["procrustes_indices"], canonical_data["procrustes_weights"], + image_width=image_width, image_height=image_height, + ) diff --git a/ComfyUI/comfy_extras/mediapipe/face_landmarker.py b/ComfyUI/comfy_extras/mediapipe/face_landmarker.py new file mode 100644 index 0000000000000000000000000000000000000000..e6b463c4c60774b60aa43efde5f4346c8d443bd8 --- /dev/null +++ b/ComfyUI/comfy_extras/mediapipe/face_landmarker.py @@ -0,0 +1,681 @@ +"""Pure-PyTorch port of MediaPipe's face_landmarker_v2_with_blendshapes.task: +BlazeFace detector → FaceMesh v2 → ARKit-52 blendshapes.""" + + +import math +from functools import lru_cache +from typing import List, Tuple + +import numpy as np +import torch +import torch.nn.functional as F +from scipy.special import expit +from torch import Tensor, nn + + +# Values below must stay verbatim with the published face_landmarker_v2 graph + +# face_blendshapes_graph.cc::kLandmarksSubsetIdxs +_BS_INPUT_INDICES: Tuple[int, ...] = ( + 0, 1, 4, 5, 6, 7, 8, 10, 13, 14, 17, 21, 33, 37, 39, 40, 46, 52, 53, 54, + 55, 58, 61, 63, 65, 66, 67, 70, 78, 80, 81, 82, 84, 87, 88, 91, 93, 95, + 103, 105, 107, 109, 127, 132, 133, 136, 144, 145, 146, 148, 149, 150, 152, + 153, 154, 155, 157, 158, 159, 160, 161, 162, 163, 168, 172, 173, 176, 178, + 181, 185, 191, 195, 197, 234, 246, 249, 251, 263, 267, 269, 270, 276, 282, + 283, 284, 285, 288, 291, 293, 295, 296, 297, 300, 308, 310, 311, 312, 314, + 317, 318, 321, 323, 324, 332, 334, 336, 338, 356, 361, 362, 365, 373, 374, + 375, 377, 378, 379, 380, 381, 382, 384, 385, 386, 387, 388, 389, 390, 397, + 398, 400, 402, 405, 409, 415, 454, 466, 468, 469, 470, 471, 472, 473, 474, + 475, 476, 477, +) + +# face_blendshapes_graph.cc::kCategoryNames +BLENDSHAPE_NAMES: Tuple[str, ...] = ( + "_neutral", "browDownLeft", "browDownRight", "browInnerUp", "browOuterUpLeft", + "browOuterUpRight", "cheekPuff", "cheekSquintLeft", "cheekSquintRight", + "eyeBlinkLeft", "eyeBlinkRight", "eyeLookDownLeft", "eyeLookDownRight", + "eyeLookInLeft", "eyeLookInRight", "eyeLookOutLeft", "eyeLookOutRight", + "eyeLookUpLeft", "eyeLookUpRight", "eyeSquintLeft", "eyeSquintRight", + "eyeWideLeft", "eyeWideRight", "jawForward", "jawLeft", "jawOpen", + "jawRight", "mouthClose", "mouthDimpleLeft", "mouthDimpleRight", + "mouthFrownLeft", "mouthFrownRight", "mouthFunnel", "mouthLeft", + "mouthLowerDownLeft", "mouthLowerDownRight", "mouthPressLeft", + "mouthPressRight", "mouthPucker", "mouthRight", "mouthRollLower", + "mouthRollUpper", "mouthShrugLower", "mouthShrugUpper", "mouthSmileLeft", + "mouthSmileRight", "mouthStretchLeft", "mouthStretchRight", + "mouthUpperUpLeft", "mouthUpperUpRight", "noseSneerLeft", "noseSneerRight", +) + +# face_detection.pbtxt — short-range BlazeFace. +_BF_NUM_LAYERS = 4 +_BF_INPUT_SIZE = 128 +_BF_STRIDES = (8, 16, 16, 16) +_BF_ANCHOR_OFFSET_X = 0.5 +_BF_ANCHOR_OFFSET_Y = 0.5 +_BF_ASPECT_RATIOS = (1.0,) +_BF_INTERP_SCALE_AR = 1.0 +_BF_BOX_SCALE = 128.0 +_BF_KP_OFFSET = 4 +_BF_SCORE_CLIP = 100.0 +_BF_MIN_SCORE = 0.5 + +# face_detection_full_range.pbtxt — 48x48 grid at stride 4, 1 anchor/cell. +_BF_FR_INPUT_SIZE = 192 +_BF_FR_GRID = 48 +_BF_FR_NUM_ANCHORS = _BF_FR_GRID * _BF_FR_GRID +_BF_FR_BOX_SCALE = 192.0 +_BF_FR_SCORE_CLIP = 100.0 + +_FM_INPUT_SIZE = 192 + +# Face ROI: 1.5xbbox rect warped anisotropically into 192x192. +_FACE_LEFT_EYE_KP = 0 +_FACE_RIGHT_EYE_KP = 1 +_FACE_ROI_SCALE_X = 1.5 +_FACE_ROI_SCALE_Y = 1.5 +_FACE_ROI_TARGET_ANGLE = 0.0 + + +def _tf_same_pad(x: Tensor, kernel: int, stride: int) -> Tensor: + """TF SAME pad (asymmetric on stride-2; PyTorch's symmetric pad undershoots by 1 px).""" + H, W = x.shape[-2], x.shape[-1] + pad_h = max(((H + stride - 1) // stride - 1) * stride + kernel - H, 0) + pad_w = max(((W + stride - 1) // stride - 1) * stride + kernel - W, 0) + if pad_h == 0 and pad_w == 0: + return x + return F.pad(x, (pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2)) + + +# BlazeFace short-range: stem 5x5/s2 → 16 BlazeBlocks → parallel heads at +# 16²x88 (2 anchors/cell) and 8²x96 (6/cell) = 896 anchors. (in, out, stride): +_BLAZEFACE_BLOCKS = [ + (24, 24, 1), (24, 28, 1), (28, 32, 2), (32, 36, 1), + (36, 42, 1), (42, 48, 2), (48, 56, 1), (56, 64, 1), + (64, 72, 1), (72, 80, 1), (80, 88, 1), (88, 96, 2), + (96, 96, 1), (96, 96, 1), (96, 96, 1), (96, 96, 1), +] + + +class BlazeFaceBlock(nn.Module): + """DW 3x3 + PW + residual. Residual max-pools on stride>1, channel-pads on out_ch>in_ch.""" + + def __init__(self, in_ch: int, out_ch: int, stride: int, device=None, dtype=None, operations=None): + super().__init__() + ops = operations if operations is not None else nn + self.in_ch, self.out_ch, self.stride = in_ch, out_ch, stride + self.depthwise = ops.Conv2d(in_ch, in_ch, 3, stride=stride, padding=0, groups=in_ch, bias=True, device=device, dtype=dtype) + self.pointwise = ops.Conv2d(in_ch, out_ch, 1, padding=0, bias=True, device=device, dtype=dtype) + + def forward(self, x: Tensor) -> Tensor: + residual = F.max_pool2d(x, 2, 2) if self.stride > 1 else x + if self.out_ch > self.in_ch: + residual = F.pad(residual, (0, 0, 0, 0, 0, self.out_ch - self.in_ch)) + x = _tf_same_pad(x, 3, self.stride) if self.stride > 1 else F.pad(x, (1, 1, 1, 1)) + return F.relu(self.pointwise(self.depthwise(x)) + residual) + + +class BlazeFace(nn.Module): + """Short-range BlazeFace: (B, 3, 128, 128) in [-1, 1] → 896 anchors x 17.""" + + def __init__(self, device=None, dtype=None, operations=None): + super().__init__() + ops = operations if operations is not None else nn + kw = dict(device=device, dtype=dtype) + self.stem = ops.Conv2d(3, 24, 5, stride=2, padding=0, bias=True, **kw) + self.blocks = nn.ModuleList(BlazeFaceBlock(i, o, s, device=device, dtype=dtype, operations=operations) + for (i, o, s) in _BLAZEFACE_BLOCKS) + # 16²x2 + 8²x6 = 512 + 384 = 896 anchors. + self.cls_16 = ops.Conv2d(88, 2, 1, padding=0, bias=True, **kw) + self.cls_8 = ops.Conv2d(96, 6, 1, padding=0, bias=True, **kw) + self.reg_16 = ops.Conv2d(88, 32, 1, padding=0, bias=True, **kw) + self.reg_8 = ops.Conv2d(96, 96, 1, padding=0, bias=True, **kw) + + def forward(self, image_chw_normalized: Tensor) -> tuple[Tensor, Tensor]: + x = F.relu(self.stem(_tf_same_pad(image_chw_normalized, 5, 2))) + # 16x16 tap is block-10 output (before the 88→96 stride-2 in block 11). + for i in range(11): + x = self.blocks[i](x) + feat_16 = x + for i in range(11, 16): + x = self.blocks[i](x) + feat_8 = x + + def flat(t, a, k): # NHWC flatten → (B, H*W*A, K) + B, _, H, W = t.shape + return t.permute(0, 2, 3, 1).reshape(B, H * W * a, k) + + cls = torch.cat([flat(self.cls_16(feat_16), 2, 1), flat(self.cls_8(feat_8), 6, 1)], dim=1) + reg = torch.cat([flat(self.reg_16(feat_16), 2, 16), flat(self.reg_8(feat_8), 6, 16)], dim=1) + return reg, cls + + +# BlazeFace full-range (face_detection_full_range_sparse.tflite): MobileNetV2-ish +# backbone + top-down FPN, 192² input → 2304 anchors at the 48x48 grid. +class FRBlock(nn.Module): + """Double inverted residual: DW → PW(mid) → DW → PW(out) [+ residual]. + + Per source tflite: dw* have no fused activation, pw1 is always ReLU, pw2 + is ReLU only when no residual (else ReLU fuses into the ADD). + """ + + def __init__(self, in_ch: int, mid_ch: int, out_ch: int, stride: int, device=None, dtype=None, operations=None): + super().__init__() + ops = operations if operations is not None else nn + kw = dict(device=device, dtype=dtype) + self.has_residual = (in_ch == out_ch and stride == 1) + self.dw1 = ops.Conv2d(in_ch, in_ch, 3, stride=stride, padding=0, groups=in_ch, bias=True, **kw) + self.pw1 = ops.Conv2d(in_ch, mid_ch, 1, padding=0, bias=True, **kw) + self.dw2 = ops.Conv2d(mid_ch, mid_ch, 3, stride=1, padding=0, groups=mid_ch, bias=True, **kw) + self.pw2 = ops.Conv2d(mid_ch, out_ch, 1, padding=0, bias=True, **kw) + + def forward(self, x: Tensor) -> Tensor: + residual = x if self.has_residual else None + x = F.relu(self.pw1(self.dw1(F.pad(x, (1, 1, 1, 1))))) + x = self.pw2(self.dw2(F.pad(x, (1, 1, 1, 1)))) + return F.relu(x + residual) if residual is not None else F.relu(x) + + +# (in_ch, mid_ch, out_ch, stride). Stages downsample 96²x32 → 48²x64 → 24²x128 +# → 12²x192 → 6²x384. Lateral taps at indices 4, 7, 10 (see _FR_LATERAL_*). +_FR_BACKBONE_BLOCKS = [ + (32, 8, 32, 1), (32, 8, 32, 1), # 96²x32 + (32, 16, 64, 2), (64, 16, 64, 1), (64, 16, 64, 1), # 48²x64 — tap[0] + (64, 32, 128, 2), (128, 32, 128, 1), (128, 32, 128, 1), # 24²x128 — tap[1] + (128, 48, 192, 2), (192, 48, 192, 1), (192, 48, 192, 1), # 12²x192 — tap[2] + (192, 96, 384, 2), (384, 96, 384, 1), (384, 96, 384, 1), (384, 96, 384, 1), # 6²x384 +] +_FR_LATERAL_TAP_INDICES = (4, 7, 10) +_FR_LATERAL_CHANNELS = ((64, 48), (128, 64), (192, 96)) # (in, out) per side-conv + +# Decoder blocks per FPN level (after upsample-and-merge with the lateral). +_FR_DECODER_BLOCKS = [ + [(96, 48, 96, 1), (96, 48, 96, 1)], # 12²x96 + [(64, 32, 64, 1), (64, 32, 64, 1)], # 24²x64 + [(48, 24, 48, 1)], # 48²x48 — feeds the heads +] + + +def _dcr_depth_to_space(t: Tensor, r: int, c_out: int) -> Tensor: + """TF DEPTH_TO_SPACE in DCR layout (input channels = (i, j, c_out)). + pixel_shuffle uses CRD which permutes output channels for c_out > 1.""" + B_, _, H_, W_ = t.shape + t = t.reshape(B_, r, r, c_out, H_, W_) + t = t.permute(0, 3, 4, 1, 5, 2).contiguous() + return t.reshape(B_, c_out, H_ * r, W_ * r) + + +class BlazeFaceFullRange(nn.Module): + """Full-range face detector: (B, 3, 192, 192) in [-1, 1] → 2304 anchors x 17 values.""" + + def __init__(self, device=None, dtype=None, operations=None): + super().__init__() + ops = operations if operations is not None else nn + kw = dict(device=device, dtype=dtype) + mk_block = lambda i, m, o, s: FRBlock(i, m, o, s, device=device, dtype=dtype, operations=operations) + self.stem = ops.Conv2d(3, 32, 3, stride=2, padding=0, bias=True, **kw) + self.backbone = nn.ModuleList(mk_block(i, m, o, s) for (i, m, o, s) in _FR_BACKBONE_BLOCKS) + self.lateral_convs = nn.ModuleList(ops.Conv2d(i, o, 1, padding=0, bias=True, **kw) for (i, o) in _FR_LATERAL_CHANNELS) + self.top_conv = ops.Conv2d(384, 96, 1, padding=0, bias=True, **kw) + self.decoder_levels = nn.ModuleList( + nn.ModuleList(mk_block(i, m, o, s) for (i, m, o, s) in lvl) for lvl in _FR_DECODER_BLOCKS + ) + # 96→64 before 12→24, 64→48 before 24→48. + self.decoder_reduce_convs = nn.ModuleList([ + ops.Conv2d(96, 64, 1, padding=0, bias=True, **kw), + ops.Conv2d(64, 48, 1, padding=0, bias=True, **kw), + ]) + # Heads mix 2x2-cell info via DW-stride-2 + depth_to_space block_size=2. + self.cls_conv = ops.Conv2d(48, 4, 1, padding=0, bias=True, **kw) + self.cls_dw = ops.Conv2d(4, 4, 3, stride=2, padding=0, groups=4, bias=True, **kw) + self.reg_conv = ops.Conv2d(48, 64, 1, padding=0, bias=True, **kw) + self.reg_dw = ops.Conv2d(64, 64, 3, stride=2, padding=0, groups=64, bias=True, **kw) + + def forward(self, image_chw_normalized: Tensor) -> tuple[Tensor, Tensor]: + # Symmetric pad-1 throughout (full-range tflite uses explicit TF PAD, not SAME). + x = F.relu(self.stem(F.pad(image_chw_normalized, (1, 1, 1, 1)))) + tap_set = set(_FR_LATERAL_TAP_INDICES) + laterals: list[Tensor] = [] + for i, blk in enumerate(self.backbone): + x = blk(x) + if i in tap_set: + laterals.append(x) + + # top_conv / lateral_convs / decoder_reduce_convs all have fused ReLU in the tflite. + p = F.relu(self.top_conv(x)) + laterals_rev = list(reversed(laterals)) + lateral_convs_rev = list(reversed(self.lateral_convs)) + for level in range(len(self.decoder_levels)): + lateral = laterals_rev[level] + p = F.interpolate(p, size=lateral.shape[-2:], mode="bilinear", align_corners=False) + p = p + F.relu(lateral_convs_rev[level](lateral)) + for blk in self.decoder_levels[level]: + p = blk(p) + if level < len(self.decoder_reduce_convs): + p = F.relu(self.decoder_reduce_convs[level](p)) + + c = self.cls_dw(F.pad(self.cls_conv(p), (1, 1, 1, 1))) + c = _dcr_depth_to_space(c, r=2, c_out=1) + r = self.reg_dw(F.pad(self.reg_conv(p), (1, 1, 1, 1))) + r = _dcr_depth_to_space(r, r=2, c_out=16) + B = c.shape[0] + cls_out = c.permute(0, 2, 3, 1).reshape(B, _BF_FR_NUM_ANCHORS, 1) + reg_out = r.permute(0, 2, 3, 1).reshape(B, _BF_FR_NUM_ANCHORS, 16) + return reg_out, cls_out + + +@lru_cache(maxsize=1) +def _blazeface_full_range_anchors() -> np.ndarray: + """2304 anchors over 48x48; anchor_w=anchor_h=1 (fixed_anchor_size).""" + feat = _BF_FR_GRID + yy, xx = np.meshgrid(np.arange(feat, dtype=np.float32), np.arange(feat, dtype=np.float32), indexing="ij") + cx, cy, ones = (xx + 0.5) / feat, (yy + 0.5) / feat, np.ones_like(xx) + return np.stack([cx, cy, ones, ones], axis=-1).reshape(_BF_FR_NUM_ANCHORS, 4) + + +def _decode_blazeface_full_range(regressors: np.ndarray, classificators: np.ndarray, + score_thresh: float = _BF_MIN_SCORE) -> np.ndarray: + """Same decode as short-range with 2304-anchor grid and box_scale=192.""" + scores = expit(np.clip(classificators[:, 0], -_BF_FR_SCORE_CLIP, _BF_FR_SCORE_CLIP)) + keep = scores >= score_thresh + if not keep.any(): + return np.empty((0, 17), dtype=np.float32) + r = regressors[keep] / _BF_FR_BOX_SCALE + a = _blazeface_full_range_anchors()[keep] + cxs, cys, aws, ahs = a[:, 0:1], a[:, 1:2], a[:, 2:3], a[:, 3:4] + xc, yc = r[:, 0:1] * aws + cxs, r[:, 1:2] * ahs + cys + w, h = r[:, 2:3] * aws, r[:, 3:4] * ahs + out = np.empty((r.shape[0], 17), dtype=np.float32) + out[:, 0:1], out[:, 1:2], out[:, 2:3], out[:, 3:4] = xc - w / 2, yc - h / 2, xc + w / 2, yc + h / 2 + out[:, 4:16:2] = r[:, _BF_KP_OFFSET::2] * aws + cxs + out[:, 5:16:2] = r[:, _BF_KP_OFFSET + 1::2] * ahs + cys + out[:, 16] = scores[keep] + return out + + +# FaceMesh (face_landmarks_detector.tflite): PReLU variant of BlazeBlock, +# 17 blocks, heads for 478x3 landmarks + presence. +_FACEMESH_BLOCKS = [ # (in_ch, out_ch, stride) + (16, 16, 1), (16, 16, 1), (16, 32, 2), (32, 32, 1), (32, 32, 1), (32, 64, 2), + (64, 64, 1), (64, 64, 1), (64, 128, 2), (128, 128, 1), (128, 128, 1), (128, 128, 2), + (128, 128, 1), (128, 128, 1), (128, 128, 2), (128, 128, 1), (128, 128, 1), +] + + +class FaceMeshBlock(nn.Module): + """PReLU BlazeBlock: PReLU between DW and PW, and after the residual add.""" + + def __init__(self, in_ch: int, out_ch: int, stride: int, device=None, dtype=None, operations=None): + super().__init__() + ops = operations if operations is not None else nn + kw = dict(device=device, dtype=dtype) + self.in_ch, self.out_ch, self.stride = in_ch, out_ch, stride + self.depthwise = ops.Conv2d(in_ch, in_ch, 3, stride=stride, padding=0, groups=in_ch, bias=True, **kw) + self.prelu_dwise = nn.PReLU(num_parameters=in_ch, **kw) + self.pointwise = ops.Conv2d(in_ch, out_ch, 1, padding=0, bias=True, **kw) + self.prelu_out = nn.PReLU(num_parameters=out_ch, **kw) + + def forward(self, x: Tensor) -> Tensor: + residual = F.max_pool2d(x, 2, 2) if self.stride > 1 else x + if self.out_ch > self.in_ch: + residual = F.pad(residual, (0, 0, 0, 0, 0, self.out_ch - self.in_ch)) + x = _tf_same_pad(x, 3, self.stride) if self.stride > 1 else F.pad(x, (1, 1, 1, 1)) + return self.prelu_out(self.pointwise(self.prelu_dwise(self.depthwise(x))) + residual) + + +class FaceMesh(nn.Module): + NUM_LANDMARKS = 478 + + def __init__(self, device=None, dtype=None, operations=None): + super().__init__() + ops = operations if operations is not None else nn + kw = dict(device=device, dtype=dtype) + self.stem = ops.Conv2d(3, 16, 3, stride=2, padding=0, bias=True, **kw) + self.prelu_stem = nn.PReLU(num_parameters=16, **kw) + self.blocks = nn.ModuleList(FaceMeshBlock(i, o, s, device=device, dtype=dtype, operations=operations) + for (i, o, s) in _FACEMESH_BLOCKS) + self.head_reduce = ops.Conv2d(128, 8, 1, padding=0, bias=True, **kw) + self.prelu_head_reduce = nn.PReLU(num_parameters=8, **kw) + self.head_block = FaceMeshBlock(8, 8, 1, device=device, dtype=dtype, operations=operations) + self.head_presence = ops.Conv2d(8, 1, 3, padding=0, bias=True, **kw) + self.head_landmarks = ops.Conv2d(8, self.NUM_LANDMARKS * 3, 3, padding=0, bias=True, **kw) + + def forward(self, face_chw_normalized: Tensor) -> tuple[Tensor, Tensor]: + """(B, 3, 192, 192) in [0, 1] → ((B, 478, 3) landmarks in 192-canonical, (B,) presence).""" + x = self.prelu_stem(self.stem(_tf_same_pad(face_chw_normalized, 3, 2))) + for blk in self.blocks: + x = blk(x) + x = self.prelu_head_reduce(self.head_reduce(x)) + x = self.head_block(x) + B = x.shape[0] + presence = self.head_presence(x).reshape(B) + lmks = self.head_landmarks(x).reshape(B, self.NUM_LANDMARKS, 3) + return lmks, presence + + +# FaceBlendshapes (MLP-Mixer "GhumMarkerPoserMlpMixerGeneral"): +# 146x2 → token-reduce 146→96 → embed 2→64 → +cls token → 4x mixer → cls→52. +_BS_NUM_INPUT_LANDMARKS = 146 +_BS_NUM_TOKENS_REDUCED = 96 +_BS_NUM_TOKENS = 97 # +1 cls +_BS_TOKEN_DIM = 64 +_BS_TOKEN_MIX_HIDDEN = 384 +_BS_CHANNEL_MIX_HIDDEN = 256 +_BS_NUM_BLENDSHAPES = 52 +_BS_LN_EPS = 1e-6 + + +class MlpMixerBlock(nn.Module): + """MLP-Mixer block: token-mixing MLP (over tokens) → channel-mixing MLP (over dim). + Both pre-LN, both residual. LN has no beta (bias=False) to match MP.""" + + def __init__(self, num_tokens: int, token_dim: int, token_hidden: int, channel_hidden: int, + device=None, dtype=None, operations=None): + super().__init__() + ops = operations if operations is not None else nn + kw = dict(device=device, dtype=dtype) + # bias=False → no LN beta (matches MP). + self.ln1 = ops.LayerNorm(token_dim, eps=_BS_LN_EPS, bias=False, **kw) + self.ln2 = ops.LayerNorm(token_dim, eps=_BS_LN_EPS, bias=False, **kw) + self.token_mlp1 = ops.Linear(num_tokens, token_hidden, bias=True, **kw) + self.token_mlp2 = ops.Linear(token_hidden, num_tokens, bias=True, **kw) + self.channel_mlp1 = ops.Linear(token_dim, channel_hidden, bias=True, **kw) + self.channel_mlp2 = ops.Linear(channel_hidden, token_dim, bias=True, **kw) + + def forward(self, x: Tensor) -> Tensor: + y = self.ln1(x).transpose(1, 2) + x = x + self.token_mlp2(F.relu(self.token_mlp1(y))).transpose(1, 2) + return x + self.channel_mlp2(F.relu(self.channel_mlp1(self.ln2(x)))) + + +class FaceBlendshapes(nn.Module): + def __init__(self, device=None, dtype=None, operations=None): + super().__init__() + ops = operations if operations is not None else nn + kw = dict(device=device, dtype=dtype) + self.token_reduce = ops.Linear(_BS_NUM_INPUT_LANDMARKS, _BS_NUM_TOKENS_REDUCED, bias=True, **kw) + self.token_embed = ops.Linear(2, _BS_TOKEN_DIM, bias=True, **kw) + self.cls_token = nn.Parameter(torch.zeros(1, 1, _BS_TOKEN_DIM, **kw)) + self.blocks = nn.ModuleList( + MlpMixerBlock(_BS_NUM_TOKENS, _BS_TOKEN_DIM, _BS_TOKEN_MIX_HIDDEN, _BS_CHANNEL_MIX_HIDDEN, + device=device, dtype=dtype, operations=operations) for _ in range(4) + ) + self.head = ops.Linear(_BS_TOKEN_DIM, _BS_NUM_BLENDSHAPES, bias=True, **kw) + + @staticmethod + def _input_normalize(landmarks_2d: Tensor) -> Tensor: + # Centroid-subtract → L2 scale → x0.5. The 0.5 is baked into training. + centroid = landmarks_2d.mean(dim=1, keepdim=True) + x = landmarks_2d - centroid + mag = torch.sqrt((x * x).sum(dim=-1, keepdim=True)) + scale = mag.mean(dim=1, keepdim=True) + return (x / scale.clamp(min=1e-12)) * 0.5 + + def forward(self, landmarks_2d: Tensor) -> Tensor: + """(B, 146, 2) → (B, 52) in [0, 1]. Input units don't matter (centroid + L2 normalize).""" + x = self._input_normalize(landmarks_2d) + x = self.token_reduce(x.transpose(1, 2)).transpose(1, 2) + x = self.token_embed(x) + cls = self.cls_token.expand(x.shape[0], -1, -1) + x = torch.cat([cls, x], dim=1) + for blk in self.blocks: + x = blk(x) + return torch.sigmoid(self.head(x[:, 0])) + + +@lru_cache(maxsize=1) +def _blazeface_anchors() -> np.ndarray: + """896 anchors per SsdAnchorsCalculator (fixed_anchor_size → anchor_w=anchor_h=1).""" + per_ar = len(_BF_ASPECT_RATIOS) + (1 if _BF_INTERP_SCALE_AR > 0 else 0) + layer_anchors: List[np.ndarray] = [] + layer = 0 + while layer < _BF_NUM_LAYERS: + stride = _BF_STRIDES[layer] + last = layer + while last < _BF_NUM_LAYERS and _BF_STRIDES[last] == stride: + last += 1 + per_cell = per_ar * (last - layer) + feat = (_BF_INPUT_SIZE + stride - 1) // stride + yy, xx = np.meshgrid(np.arange(feat, dtype=np.float32), np.arange(feat, dtype=np.float32), indexing="ij") + cx, cy, ones = (xx + _BF_ANCHOR_OFFSET_X) / feat, (yy + _BF_ANCHOR_OFFSET_Y) / feat, np.ones_like(xx) + cell = np.stack([cx, cy, ones, ones], axis=-1).reshape(-1, 4) + layer_anchors.append(np.repeat(cell, per_cell, axis=0)) + layer = last + out = np.concatenate(layer_anchors, axis=0) + assert out.shape == (896, 4), out.shape + return out + + +def _decode_blazeface(regressors: np.ndarray, classificators: np.ndarray, + score_thresh: float = _BF_MIN_SCORE) -> np.ndarray: + """Decode (regs (896,16), cls (896,1)) → (N, 17) = [xyxy, kp0x..kp5y, score] in [0, 1].""" + scores = expit(np.clip(classificators[:, 0], -_BF_SCORE_CLIP, _BF_SCORE_CLIP)) + keep = scores >= score_thresh + if not keep.any(): + return np.empty((0, 17), dtype=np.float32) + r = regressors[keep] / _BF_BOX_SCALE + a = _blazeface_anchors()[keep] # (N, 4) cx, cy, 1, 1 + cxs, cys, aws, ahs = a[:, 0:1], a[:, 1:2], a[:, 2:3], a[:, 3:4] + xc, yc = r[:, 0:1] * aws + cxs, r[:, 1:2] * ahs + cys + w, h = r[:, 2:3] * aws, r[:, 3:4] * ahs + out = np.empty((r.shape[0], 17), dtype=np.float32) + out[:, 0:1], out[:, 1:2], out[:, 2:3], out[:, 3:4] = xc - w / 2, yc - h / 2, xc + w / 2, yc + h / 2 + out[:, 4:16:2] = r[:, _BF_KP_OFFSET::2] * aws + cxs + out[:, 5:16:2] = r[:, _BF_KP_OFFSET + 1::2] * ahs + cys + out[:, 16] = scores[keep] + return out + + +def _weighted_nms(detections: np.ndarray, iou_thresh: float = 0.5) -> np.ndarray: + """MP weighted NMS — kept boxes are score-weighted averages of overlapping detections.""" + if detections.shape[0] == 0: + return detections + dets = detections[np.argsort(-detections[:, 16])] + N = dets.shape[0] + areas = np.clip(dets[:, 2] - dets[:, 0], 0, None) * np.clip(dets[:, 3] - dets[:, 1], 0, None) + kept: List[np.ndarray] = [] + used = np.zeros(N, dtype=bool) + for i in range(N): + if used[i]: + continue + ax1, ay1, ax2, ay2 = dets[i, 0:4] + merge_idx = [i] + for j in range(i + 1, N): + if used[j]: + continue + bx1, by1, bx2, by2 = dets[j, 0:4] + iw = max(0.0, min(ax2, bx2) - max(ax1, bx1)) + ih = max(0.0, min(ay2, by2) - max(ay1, by1)) + inter = iw * ih + union = areas[i] + areas[j] - inter + if union > 0 and inter / union > iou_thresh: # strict > matches MP + merge_idx.append(j) + used[j] = True + used[i] = True + cluster = dets[merge_idx] + ws = cluster[:, 16:17] + ws_sum = ws.sum() + merged = np.copy(cluster[0]) + if ws_sum > 0: + merged[:16] = (cluster[:, :16] * ws).sum(axis=0) / ws_sum + kept.append(merged) + return np.stack(kept, axis=0) if kept else np.empty((0, 17), dtype=np.float32) + + +def _detection_to_face_rect(detection: np.ndarray, image_w: int, image_h: int) -> Tuple[float, float, float, float, float]: + """Detection (normalized) → rotated 1.5xbbox ROI in image pixels (anisotropic).""" + xmin, ymin, xmax, ymax = detection[0:4] + lx = detection[4 + _FACE_LEFT_EYE_KP * 2 + 0] * image_w + ly = detection[4 + _FACE_LEFT_EYE_KP * 2 + 1] * image_h + rx = detection[4 + _FACE_RIGHT_EYE_KP * 2 + 0] * image_w + ry = detection[4 + _FACE_RIGHT_EYE_KP * 2 + 1] * image_h + # Image-y-down convention: angle = target - atan2(-dy, dx). + angle = _FACE_ROI_TARGET_ANGLE - math.atan2(ly - ry, rx - lx) + return (float((xmin + xmax) * 0.5 * image_w), + float((ymin + ymax) * 0.5 * image_h), + float((xmax - xmin) * image_w * _FACE_ROI_SCALE_X), + float((ymax - ymin) * image_h * _FACE_ROI_SCALE_Y), + float(angle)) + + +def _sample_warp(image_chw: Tensor, src_x: Tensor, src_y: Tensor, padding_mode: str) -> Tensor: + """Bilinear-sample image_chw at corner-aligned (src_x, src_y).""" + H, W = int(image_chw.shape[-2]), int(image_chw.shape[-1]) + grid = torch.stack([(2.0 * src_x + 1.0) / W - 1.0, + (2.0 * src_y + 1.0) / H - 1.0], dim=-1).unsqueeze(0) + return F.grid_sample(image_chw.unsqueeze(0), grid, mode="bilinear", + align_corners=False, padding_mode=padding_mode).squeeze(0) + + +def _warp_face_crop(image_chw: Tensor, cx: float, cy: float, width: float, height: float, + angle: float, output_size: int = _FM_INPUT_SIZE) -> Tensor: + """Rotated rect → output_size² with BORDER_REPLICATE. image_chw must be in [0, 1].""" + s_x, s_y = width / output_size, height / output_size + cos_a, sin_a = math.cos(angle), math.sin(angle) + arange = torch.arange(output_size, dtype=image_chw.dtype, device=image_chw.device) - output_size * 0.5 + v_grid, u_grid = torch.meshgrid(arange, arange, indexing="ij") + src_x = cx + u_grid * s_x * cos_a - v_grid * s_y * sin_a + src_y = cy + u_grid * s_x * sin_a + v_grid * s_y * cos_a + return _sample_warp(image_chw, src_x, src_y, "border") + + +def _blazeface_input_warp(image_chw_raw: Tensor, target: int = _BF_INPUT_SIZE) -> Tuple[Tensor, float, float, float]: + """Centered max(W,H) square → target² with BORDER_ZERO + [-1, 1] norm. + + Sub-pixel grid_sample matters; integer-pad-then-resize drifts the bbox ~5%. + Returns (warped, sub_rect_cx, sub_rect_cy, sub_rect_size) — the triplet maps + tensor-normalized [0,1] detections back to image pixels. + """ + H, W = int(image_chw_raw.shape[1]), int(image_chw_raw.shape[2]) + sub_rect_size = float(max(W, H)) + sub_rect_cx, sub_rect_cy = W * 0.5, H * 0.5 + s = sub_rect_size / target + arange = torch.arange(target, dtype=image_chw_raw.dtype, device=image_chw_raw.device) - target * 0.5 + v_grid, u_grid = torch.meshgrid(arange, arange, indexing="ij") + out = _sample_warp(image_chw_raw, sub_rect_cx + u_grid * s, sub_rect_cy + v_grid * s, "zeros") + return (out / 127.5) - 1.0, sub_rect_cx, sub_rect_cy, sub_rect_size + + +class FaceLandmarker(nn.Module): + """BlazeFace → FaceMesh v2 → blendshapes. `detector_variant` selects 'short' + (128², ≤2m) or 'full' (192² FPN, ≤5m). State dict uses inner-module prefixes + `detector.*` / `mesh.*` / `blendshapes.*`; the outer FaceLandmarkerModel + wrapper rewrites `detector_{variant}.*` keys to `detector.*` before loading. + """ + + def __init__(self, device=None, dtype=None, operations=None, detector_variant: str = "short"): + super().__init__() + det_cls = {"short": BlazeFace, "full": BlazeFaceFullRange}.get(detector_variant) + + self.detector_variant = detector_variant + self.detector = det_cls(device=device, dtype=dtype, operations=operations) + self.mesh = FaceMesh(device=device, dtype=dtype, operations=operations) + self.blendshapes = FaceBlendshapes(device=device, dtype=dtype, operations=operations) + self.register_buffer("_bs_idx", torch.tensor(_BS_INPUT_INDICES, dtype=torch.long), persistent=False) + + def run_detector_batch(self, images_rgb_uint8: List[np.ndarray], + score_thresh: float = _BF_MIN_SCORE, + iou_thresh: float = 0.5): + """Batched detector pass. Returns (img_raws, sub_rects, sizes, per_frame_decoded) + where per_frame_decoded[b] is (N, 17) in tensor-normalized [0,1] coords.""" + if not images_rgb_uint8: + return [], [], [], [] + device, dtype = self.detector.stem.weight.device, self.detector.stem.weight.dtype + det_input_size, decode_fn = ((_BF_FR_INPUT_SIZE, _decode_blazeface_full_range) + if self.detector_variant == "full" + else (_BF_INPUT_SIZE, _decode_blazeface)) + + # Same-size frames: stack once and transfer once. Variable size falls back + # to per-image (only triggers for SAM3DBody's head crops). + sizes = [tuple(img.shape[:2]) for img in images_rgb_uint8] + if len(set(sizes)) == 1: + batch_chw = torch.from_numpy(np.stack(images_rgb_uint8, axis=0)).to(device, dtype).movedim(-1, -3).contiguous() + img_raws = [batch_chw[bi] for bi in range(batch_chw.shape[0])] + else: + img_raws = [torch.from_numpy(img).to(device, dtype).movedim(-1, -3).contiguous() for img in images_rgb_uint8] + + warps = [_blazeface_input_warp(img_raw, det_input_size) for img_raw in img_raws] + det_crops = [w[0] for w in warps] + sub_rects = [(w[1], w[2], w[3]) for w in warps] + + regs_b, cls_b = self.detector(torch.stack(det_crops, dim=0)) + regs_np, cls_np = regs_b.float().cpu().numpy(), cls_b.float().cpu().numpy() + per_frame = [] + for b in range(len(images_rgb_uint8)): + decoded = decode_fn(regs_np[b], cls_np[b], score_thresh=score_thresh) + per_frame.append(_weighted_nms(decoded, iou_thresh=iou_thresh) if decoded.shape[0] > 0 else decoded) + return img_raws, sub_rects, sizes, per_frame + + def detect_batch(self, images_rgb_uint8: List[np.ndarray], num_faces: int = 1, + score_thresh: float = _BF_MIN_SCORE) -> List[List[dict]]: + """Full pipeline batched across `images_rgb_uint8`. Returns one face-dict + list per image (empty if nothing detected). Face dict: + bbox_xyxy (4,) image pixels, blendshapes {52} ∈ [0,1], + landmarks_xy (478, 2) image pixels, landmarks_3d (478, 3) in + 192-canonical (pre-transformation) units, presence float (raw logit). + """ + img_raws, sub_rects, sizes, per_frame_dets = self.run_detector_batch( + images_rgb_uint8, score_thresh=score_thresh, + ) + # tensor-normalized → image-normalized [0,1] for _detection_to_face_rect. + for b, decoded in enumerate(per_frame_dets): + if decoded.shape[0] == 0: + continue + cx, cy, size = sub_rects[b] + H, W = sizes[b] + sx0, sy0 = cx - size * 0.5, cy - size * 0.5 + decoded[:, 0:16:2] = (sx0 + size * decoded[:, 0:16:2]) / W + decoded[:, 1:16:2] = (sy0 + size * decoded[:, 1:16:2]) / H + if num_faces > 0: + per_frame_dets[b] = decoded[: int(num_faces)] + + # Collect every detected face across all frames into one mesh input. + face_params: List[Tuple[int, float, float, float, float, float, float]] = [] + mesh_crops: List[Tensor] = [] + for b, dets in enumerate(per_frame_dets): + if dets.shape[0] == 0: + continue + H, W = sizes[b] + img_for_mesh = img_raws[b] / 255.0 + for det in dets: + cx, cy, w, h, angle = _detection_to_face_rect(det, W, H) + mesh_crops.append(_warp_face_crop(img_for_mesh, cx, cy, w, h, angle, _FM_INPUT_SIZE)) + face_params.append((b, float(det[16]), cx, cy, w, h, angle)) + + results: List[List[dict]] = [[] for _ in range(len(images_rgb_uint8))] + if not mesh_crops: + return results + + lmks_canon_b, presence_b = self.mesh(torch.stack(mesh_crops, dim=0)) + bs_out_b = self.blendshapes(lmks_canon_b[:, self._bs_idx, :2]) + + # Batched canonical→image affine + params_t = torch.tensor( + [(cx, cy, w, h, math.cos(a), math.sin(a)) for (_b, _s, cx, cy, w, h, a) in face_params], + device=lmks_canon_b.device, dtype=lmks_canon_b.dtype, + ) + cxs, cys, ws, hs, cos_a, sin_a = params_t.unbind(dim=1) + inv = 1.0 / _FM_INPUT_SIZE + u = lmks_canon_b[..., 0] - _FM_INPUT_SIZE * 0.5 + v = lmks_canon_b[..., 1] - _FM_INPUT_SIZE * 0.5 + lmks_xy_t = torch.stack([ + cxs[:, None] + u * (ws * inv * cos_a)[:, None] - v * (hs * inv * sin_a)[:, None], + cys[:, None] + u * (ws * inv * sin_a)[:, None] + v * (hs * inv * cos_a)[:, None], + ], dim=-1) + + lmks_xy_np = lmks_xy_t.float().cpu().numpy() + lmks_canon_np = lmks_canon_b.float().cpu().numpy() + presence_np = presence_b.float().cpu().numpy() + bs_np = bs_out_b.float().cpu().numpy() + + for i, (b, score, *_) in enumerate(face_params): + lmks_xy = lmks_xy_np[i] + mn, mx = lmks_xy.min(0), lmks_xy.max(0) + results[b].append({ + "bbox_xyxy": np.array([mn[0], mn[1], mx[0], mx[1]], dtype=np.float32), + "blendshapes": dict(zip(BLENDSHAPE_NAMES, bs_np[i].tolist())), + "landmarks_xy": lmks_xy, + "landmarks_3d": lmks_canon_np[i], + "presence": float(presence_np[i]), + "score": score, + }) + return results diff --git a/ComfyUI/comfy_extras/nodes_ace.py b/ComfyUI/comfy_extras/nodes_ace.py new file mode 100644 index 0000000000000000000000000000000000000000..eaf234d5be8e59200891f4053344ccd58250151f --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_ace.py @@ -0,0 +1,145 @@ +import torch +from typing_extensions import override + +import comfy.model_management +import node_helpers +from comfy_api.latest import ComfyExtension, IO + + +class TextEncodeAceStepAudio(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="TextEncodeAceStepAudio", + category="model/conditioning/ace", + inputs=[ + IO.Clip.Input("clip"), + IO.String.Input("tags", multiline=True, dynamic_prompts=True), + IO.String.Input("lyrics", multiline=True, dynamic_prompts=True), + IO.Float.Input("lyrics_strength", default=1.0, min=0.0, max=10.0, step=0.01), + ], + outputs=[IO.Conditioning.Output()], + ) + + @classmethod + def execute(cls, clip, tags, lyrics, lyrics_strength) -> IO.NodeOutput: + tokens = clip.tokenize(tags, lyrics=lyrics) + conditioning = clip.encode_from_tokens_scheduled(tokens) + conditioning = node_helpers.conditioning_set_values(conditioning, {"lyrics_strength": lyrics_strength}) + return IO.NodeOutput(conditioning) + +class TextEncodeAceStepAudio15(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="TextEncodeAceStepAudio1.5", + category="model/conditioning/ace", + inputs=[ + IO.Clip.Input("clip"), + IO.String.Input("tags", multiline=True, dynamic_prompts=True), + IO.String.Input("lyrics", multiline=True, dynamic_prompts=True), + IO.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff, control_after_generate=True), + IO.Int.Input("bpm", default=120, min=10, max=300), + IO.Float.Input("duration", default=120.0, min=0.0, max=2000.0, step=0.1), + IO.Combo.Input("timesignature", options=['2', '3', '4', '6']), + IO.Combo.Input("language", options=['ar', 'az', 'bg', 'bn', 'ca', 'cs', 'da', 'de', 'el', 'en', 'es', 'fa', 'fi', 'fr', 'he', 'hi', 'hr', 'ht', 'hu', 'id', 'is', 'it', 'ja', 'ko', 'la', 'lt', 'ms', 'ne', 'nl', 'no', 'pa', 'pl', 'pt', 'ro', 'ru', 'sa', 'sk', 'sr', 'sv', 'sw', 'ta', 'te', 'th', 'tl', 'tr', 'uk', 'ur', 'vi', 'yue', 'zh', 'unknown'], default='en'), + IO.Combo.Input("keyscale", options=[f"{root} {quality}" for quality in ["major", "minor"] for root in ["C", "C#", "Db", "D", "D#", "Eb", "E", "F", "F#", "Gb", "G", "G#", "Ab", "A", "A#", "Bb", "B"]]), + IO.Boolean.Input("generate_audio_codes", default=True, tooltip="Enable the LLM that generates audio codes. This can be slow but will increase the quality of the generated audio. Turn this off if you are giving the model an audio reference.", advanced=True), + IO.Float.Input("cfg_scale", default=2.0, min=0.0, max=100.0, step=0.1, advanced=True), + IO.Float.Input("temperature", default=0.85, min=0.0, max=2.0, step=0.01, advanced=True), + IO.Float.Input("top_p", default=0.9, min=0.0, max=2000.0, step=0.01, advanced=True), + IO.Int.Input("top_k", default=0, min=0, max=100, advanced=True), + IO.Float.Input("min_p", default=0.000, min=0.0, max=1.0, step=0.001, advanced=True), + ], + outputs=[IO.Conditioning.Output()], + ) + + @classmethod + def execute(cls, clip, tags, lyrics, seed, bpm, duration, timesignature, language, keyscale, generate_audio_codes, cfg_scale, temperature, top_p, top_k, min_p) -> IO.NodeOutput: + tokens = clip.tokenize(tags, lyrics=lyrics, bpm=bpm, duration=duration, timesignature=int(timesignature), language=language, keyscale=keyscale, seed=seed, generate_audio_codes=generate_audio_codes, cfg_scale=cfg_scale, temperature=temperature, top_p=top_p, top_k=top_k, min_p=min_p) + conditioning = clip.encode_from_tokens_scheduled(tokens) + return IO.NodeOutput(conditioning) + + +class EmptyAceStepLatentAudio(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="EmptyAceStepLatentAudio", + display_name="Empty Ace Step 1.0 Latent Audio", + category="model/latent/ace", + inputs=[ + IO.Float.Input("seconds", default=120.0, min=1.0, max=1000.0, step=0.1), + IO.Int.Input( + "batch_size", default=1, min=1, max=4096, tooltip="The number of latent images in the batch." + ), + ], + outputs=[IO.Latent.Output()], + ) + + @classmethod + def execute(cls, seconds, batch_size) -> IO.NodeOutput: + length = int(seconds * 44100 / 512 / 8) + latent = torch.zeros([batch_size, 8, 16, length], device=comfy.model_management.intermediate_device(), dtype=comfy.model_management.intermediate_dtype()) + return IO.NodeOutput({"samples": latent, "type": "audio"}) + + +class EmptyAceStep15LatentAudio(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="EmptyAceStep1.5LatentAudio", + display_name="Empty Ace Step 1.5 Latent Audio", + category="model/latent/ace", + inputs=[ + IO.Float.Input("seconds", default=120.0, min=1.0, max=1000.0, step=0.01), + IO.Int.Input( + "batch_size", default=1, min=1, max=4096, tooltip="The number of latent images in the batch." + ), + ], + outputs=[IO.Latent.Output()], + ) + + @classmethod + def execute(cls, seconds, batch_size) -> IO.NodeOutput: + length = round((seconds * 48000 / 1920)) + latent = torch.zeros([batch_size, 64, length], device=comfy.model_management.intermediate_device(), dtype=comfy.model_management.intermediate_dtype()) + return IO.NodeOutput({"samples": latent, "type": "audio", "downscale_ratio_temporal": 1764}) + +class ReferenceAudio(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="ReferenceTimbreAudio", + display_name="Set Reference Audio", + category="model/conditioning", + is_experimental=True, + description="This node sets the reference audio for ace step 1.5", + inputs=[ + IO.Conditioning.Input("conditioning"), + IO.Latent.Input("latent", optional=True), + ], + outputs=[ + IO.Conditioning.Output(), + ] + ) + + @classmethod + def execute(cls, conditioning, latent=None) -> IO.NodeOutput: + if latent is not None: + conditioning = node_helpers.conditioning_set_values(conditioning, {"reference_audio_timbre_latents": [latent["samples"]]}, append=True) + return IO.NodeOutput(conditioning) + +class AceExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[IO.ComfyNode]]: + return [ + TextEncodeAceStepAudio, + EmptyAceStepLatentAudio, + TextEncodeAceStepAudio15, + EmptyAceStep15LatentAudio, + ReferenceAudio, + ] + +async def comfy_entrypoint() -> AceExtension: + return AceExtension() diff --git a/ComfyUI/comfy_extras/nodes_advanced_samplers.py b/ComfyUI/comfy_extras/nodes_advanced_samplers.py new file mode 100644 index 0000000000000000000000000000000000000000..77a561e30a72b511935c9999431a6e20a9bcdca5 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_advanced_samplers.py @@ -0,0 +1,153 @@ +import numpy as np +import torch +from tqdm.auto import trange +from typing_extensions import override + +import comfy.model_patcher +import comfy.samplers +import comfy.utils +from comfy.k_diffusion.sampling import to_d +from comfy_api.latest import ComfyExtension, io + + +@torch.no_grad() +def sample_lcm_upscale(model, x, sigmas, extra_args=None, callback=None, disable=None, total_upscale=2.0, upscale_method="bislerp", upscale_steps=None): + extra_args = {} if extra_args is None else extra_args + + if upscale_steps is None: + upscale_steps = max(len(sigmas) // 2 + 1, 2) + else: + upscale_steps += 1 + upscale_steps = min(upscale_steps, len(sigmas) + 1) + + upscales = np.linspace(1.0, total_upscale, upscale_steps)[1:] + + orig_shape = x.size() + s_in = x.new_ones([x.shape[0]]) + for i in trange(len(sigmas) - 1, disable=disable): + denoised = model(x, sigmas[i] * s_in, **extra_args) + if callback is not None: + callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised}) + + x = denoised + if i < len(upscales): + x = comfy.utils.common_upscale(x, round(orig_shape[-1] * upscales[i]), round(orig_shape[-2] * upscales[i]), upscale_method, "disabled") + + if sigmas[i + 1] > 0: + x += sigmas[i + 1] * torch.randn_like(x) + return x + + +class SamplerLCMUpscale(io.ComfyNode): + UPSCALE_METHODS = ["bislerp", "nearest-exact", "bilinear", "area", "bicubic"] + + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="SamplerLCMUpscale", + category="model/sampling/samplers", + inputs=[ + io.Float.Input("scale_ratio", default=1.0, min=0.1, max=20.0, step=0.01, advanced=True), + io.Int.Input("scale_steps", default=-1, min=-1, max=1000, step=1, advanced=True), + io.Combo.Input("upscale_method", options=cls.UPSCALE_METHODS), + ], + outputs=[io.Sampler.Output()], + ) + + @classmethod + def execute(cls, scale_ratio, scale_steps, upscale_method) -> io.NodeOutput: + if scale_steps < 0: + scale_steps = None + sampler = comfy.samplers.KSAMPLER(sample_lcm_upscale, extra_options={"total_upscale": scale_ratio, "upscale_steps": scale_steps, "upscale_method": upscale_method}) + return io.NodeOutput(sampler) + + +@torch.no_grad() +def sample_euler_pp(model, x, sigmas, extra_args=None, callback=None, disable=None): + extra_args = {} if extra_args is None else extra_args + + temp = [0] + def post_cfg_function(args): + temp[0] = args["uncond_denoised"] + return args["denoised"] + + model_options = extra_args.get("model_options", {}).copy() + extra_args["model_options"] = comfy.model_patcher.set_model_options_post_cfg_function(model_options, post_cfg_function, disable_cfg1_optimization=True) + + s_in = x.new_ones([x.shape[0]]) + for i in trange(len(sigmas) - 1, disable=disable): + sigma_hat = sigmas[i] + denoised = model(x, sigma_hat * s_in, **extra_args) + d = to_d(x - denoised + temp[0], sigmas[i], denoised) + if callback is not None: + callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigma_hat, 'denoised': denoised}) + dt = sigmas[i + 1] - sigma_hat + x = x + d * dt + return x + + +class SamplerLCM(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="SamplerLCM", + category="model/sampling/samplers", + description=("LCM sampler with tunable per-step noise. s_noise is a multiplier on the model's training noise scale"), + inputs=[ + io.Float.Input("s_noise", default=1.0, min=0.0, max=64.0, step=0.01, + tooltip="Per-step noise multiplier at the first step (1.0 = match training)."), + io.Float.Input("s_noise_end", default=1.0, min=0.0, max=64.0, step=0.01, + tooltip="Per-step noise multiplier at the last step. Set equal to s_noise for a constant schedule."), + io.Float.Input("noise_clip_std", default=0.0, min=0.0, max=10.0, step=0.01, + tooltip="Clamp per-step noise to +/- N*std. 0 disables."), + ], + outputs=[io.Sampler.Output()], + ) + + @classmethod + def execute(cls, s_noise, s_noise_end, noise_clip_std) -> io.NodeOutput: + sampler = comfy.samplers.ksampler( + "lcm", + { + "s_noise": float(s_noise), + "s_noise_end": float(s_noise_end), + "noise_clip_std": float(noise_clip_std), + }, + ) + return io.NodeOutput(sampler) + + +class SamplerEulerCFGpp(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="SamplerEulerCFGpp", + display_name="SamplerEulerCFG++", + category="experimental", # "sampling/samplers" + inputs=[ + io.Combo.Input("version", options=["regular", "alternative"], advanced=True), + ], + outputs=[io.Sampler.Output()], + is_experimental=True, + ) + + @classmethod + def execute(cls, version) -> io.NodeOutput: + if version == "alternative": + sampler = comfy.samplers.KSAMPLER(sample_euler_pp) + else: + sampler = comfy.samplers.ksampler("euler_cfg_pp") + return io.NodeOutput(sampler) + + +class AdvancedSamplersExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + SamplerLCMUpscale, + SamplerLCM, + SamplerEulerCFGpp, + ] + +async def comfy_entrypoint() -> AdvancedSamplersExtension: + return AdvancedSamplersExtension() diff --git a/ComfyUI/comfy_extras/nodes_align_your_steps.py b/ComfyUI/comfy_extras/nodes_align_your_steps.py new file mode 100644 index 0000000000000000000000000000000000000000..f89a809bbee5acf8dd9eea0dd62f346c1ac44c27 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_align_your_steps.py @@ -0,0 +1,70 @@ +#from: https://research.nvidia.com/labs/toronto-ai/AlignYourSteps/howto.html +import numpy as np +import torch +from typing_extensions import override + +from comfy_api.latest import ComfyExtension, io + + +def loglinear_interp(t_steps, num_steps): + """ + Performs log-linear interpolation of a given array of decreasing numbers. + """ + xs = np.linspace(0, 1, len(t_steps)) + ys = np.log(t_steps[::-1]) + + new_xs = np.linspace(0, 1, num_steps) + new_ys = np.interp(new_xs, xs, ys) + + interped_ys = np.exp(new_ys)[::-1].copy() + return interped_ys + +NOISE_LEVELS = {"SD1": [14.6146412293, 6.4745760956, 3.8636745985, 2.6946151520, 1.8841921177, 1.3943805092, 0.9642583904, 0.6523686016, 0.3977456272, 0.1515232662, 0.0291671582], + "SDXL":[14.6146412293, 6.3184485287, 3.7681790315, 2.1811480769, 1.3405244945, 0.8620721141, 0.5550693289, 0.3798540708, 0.2332364134, 0.1114188177, 0.0291671582], + "SVD": [700.00, 54.5, 15.886, 7.977, 4.248, 1.789, 0.981, 0.403, 0.173, 0.034, 0.002]} + +class AlignYourStepsScheduler(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="AlignYourStepsScheduler", + search_aliases=["AYS scheduler"], + category="model/sampling/schedulers", + inputs=[ + io.Combo.Input("model_type", options=["SD1", "SDXL", "SVD"]), + io.Int.Input("steps", default=10, min=1, max=10000), + io.Float.Input("denoise", default=1.0, min=0.0, max=1.0, step=0.01), + ], + outputs=[io.Sigmas.Output()], + ) + + def get_sigmas(self, model_type, steps, denoise): + # Deprecated: use the V3 schema's `execute` method instead of this. + return AlignYourStepsScheduler().execute(model_type, steps, denoise).result + + @classmethod + def execute(cls, model_type, steps, denoise) -> io.NodeOutput: + total_steps = steps + if denoise < 1.0: + if denoise <= 0.0: + return io.NodeOutput(torch.FloatTensor([])) + total_steps = round(steps * denoise) + + sigmas = NOISE_LEVELS[model_type][:] + if (steps + 1) != len(sigmas): + sigmas = loglinear_interp(sigmas, steps + 1) + + sigmas = sigmas[-(total_steps + 1):] + sigmas[-1] = 0 + return io.NodeOutput(torch.FloatTensor(sigmas)) + + +class AlignYourStepsExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + AlignYourStepsScheduler, + ] + +async def comfy_entrypoint() -> AlignYourStepsExtension: + return AlignYourStepsExtension() diff --git a/ComfyUI/comfy_extras/nodes_apg.py b/ComfyUI/comfy_extras/nodes_apg.py new file mode 100644 index 0000000000000000000000000000000000000000..6e69b73f702740e39021620e0072818ac6b14395 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_apg.py @@ -0,0 +1,110 @@ +import torch +from typing_extensions import override + +from comfy_api.latest import ComfyExtension, io + + +def project(v0, v1): + v1 = torch.nn.functional.normalize(v1, dim=[-1, -2, -3]) + v0_parallel = (v0 * v1).sum(dim=[-1, -2, -3], keepdim=True) * v1 + v0_orthogonal = v0 - v0_parallel + return v0_parallel, v0_orthogonal + +class APG(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="APG", + display_name="Adaptive Projected Guidance", + category="model/sampling/custom", + inputs=[ + io.Model.Input("model"), + io.Float.Input( + "eta", + default=1.0, + min=-10.0, + max=10.0, + step=0.01, + tooltip="Controls the scale of the parallel guidance vector. Default CFG behavior at a setting of 1.", + advanced=True, + ), + io.Float.Input( + "norm_threshold", + default=5.0, + min=0.0, + max=50.0, + step=0.1, + tooltip="Normalize guidance vector to this value, normalization disable at a setting of 0.", + advanced=True, + ), + io.Float.Input( + "momentum", + default=0.0, + min=-5.0, + max=1.0, + step=0.01, + tooltip="Controls a running average of guidance during diffusion, disabled at a setting of 0.", + advanced=True, + ), + ], + outputs=[io.Model.Output()], + ) + + @classmethod + def execute(cls, model, eta, norm_threshold, momentum) -> io.NodeOutput: + running_avg = 0 + prev_sigma = None + + def pre_cfg_function(args): + nonlocal running_avg, prev_sigma + + if len(args["conds_out"]) == 1: + return args["conds_out"] + + cond = args["conds_out"][0] + uncond = args["conds_out"][1] + sigma = args["sigma"][0] + cond_scale = args["cond_scale"] + + if prev_sigma is not None and sigma > prev_sigma: + running_avg = 0 + prev_sigma = sigma + + guidance = cond - uncond + + if momentum != 0: + if not torch.is_tensor(running_avg): + running_avg = guidance + else: + running_avg = momentum * running_avg + guidance + guidance = running_avg + + if norm_threshold > 0: + guidance_norm = guidance.norm(p=2, dim=[-1, -2, -3], keepdim=True) + scale = torch.minimum( + torch.ones_like(guidance_norm), + norm_threshold / guidance_norm + ) + guidance = guidance * scale + + guidance_parallel, guidance_orthogonal = project(guidance, cond) + modified_guidance = guidance_orthogonal + eta * guidance_parallel + + modified_cond = (uncond + modified_guidance) + (cond - uncond) / cond_scale + + return [modified_cond, uncond] + args["conds_out"][2:] + + m = model.clone() + m.set_model_sampler_pre_cfg_function(pre_cfg_function) + return io.NodeOutput(m) + + +class ApgExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + APG, + ] + +async def comfy_entrypoint() -> ApgExtension: + return ApgExtension() diff --git a/ComfyUI/comfy_extras/nodes_ar_video.py b/ComfyUI/comfy_extras/nodes_ar_video.py new file mode 100644 index 0000000000000000000000000000000000000000..9d8f64b208e2e00c63aa439fe6ce0309d3655d29 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_ar_video.py @@ -0,0 +1,136 @@ +""" +ComfyUI nodes for autoregressive video generation (Causal Forcing, Self-Forcing, etc.). + - EmptyARVideoLatent: create 5D [B, C, T, H, W] video latent tensors + - SamplerARVideo: SAMPLER for the block-by-block autoregressive denoising loop + - ARVideoI2V: image-to-video conditioning for AR models (seeds KV cache with start image) +""" + +import torch +from typing_extensions import override + +import comfy.model_management +import comfy.samplers +import comfy.utils +from comfy_api.latest import ComfyExtension, io + + +class EmptyARVideoLatent(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="EmptyARVideoLatent", + category="model/latent/autoregressive", + inputs=[ + io.Int.Input("width", default=832, min=16, max=8192, step=16), + io.Int.Input("height", default=480, min=16, max=8192, step=16), + io.Int.Input("length", default=81, min=1, max=1024, step=4), + io.Int.Input("batch_size", default=1, min=1, max=64), + ], + outputs=[ + io.Latent.Output(display_name="LATENT"), + ], + ) + + @classmethod + def execute(cls, width, height, length, batch_size) -> io.NodeOutput: + lat_t = ((length - 1) // 4) + 1 + latent = torch.zeros( + [batch_size, 16, lat_t, height // 8, width // 8], + device=comfy.model_management.intermediate_device(), + ) + return io.NodeOutput({"samples": latent}) + + +class SamplerARVideo(io.ComfyNode): + """Sampler for autoregressive video models (Causal Forcing, Self-Forcing). + + All AR-loop parameters are owned by this node so they live in the workflow. + Add new widgets here as the AR sampler grows new options. + """ + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SamplerARVideo", + display_name="Sampler AR Video", + category="model/sampling/samplers", + inputs=[ + io.Int.Input( + "num_frame_per_block", + default=1, min=1, max=64, + tooltip="Frames per autoregressive block. 1 = framewise, " + "3 = chunkwise. Must match the checkpoint's training mode.", + ), + ], + outputs=[io.Sampler.Output()], + ) + + @classmethod + def execute(cls, num_frame_per_block) -> io.NodeOutput: + extra_options = { + "num_frame_per_block": num_frame_per_block, + } + return io.NodeOutput(comfy.samplers.ksampler("ar_video", extra_options)) + + +class ARVideoI2V(io.ComfyNode): + """Image-to-video setup for AR video models (Causal Forcing, Self-Forcing). + + VAE-encodes the start image and stores it in the model's transformer_options + so that sample_ar_video can seed the KV cache before denoising. + Uses the same T2V model checkpoint -- no separate I2V architecture needed. + """ + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ARVideoI2V", + category="model/conditioning/autoregressive", + inputs=[ + io.Model.Input("model"), + io.Vae.Input("vae"), + io.Image.Input("start_image"), + io.Int.Input("width", default=832, min=16, max=8192, step=16), + io.Int.Input("height", default=480, min=16, max=8192, step=16), + io.Int.Input("length", default=81, min=1, max=1024, step=4), + io.Int.Input("batch_size", default=1, min=1, max=64), + ], + outputs=[ + io.Model.Output(display_name="MODEL"), + io.Latent.Output(display_name="LATENT"), + ], + ) + + @classmethod + def execute(cls, model, vae, start_image, width, height, length, batch_size) -> io.NodeOutput: + start_image = comfy.utils.common_upscale( + start_image[:1].movedim(-1, 1), width, height, "bilinear", "center" + ).movedim(1, -1) + + initial_latent = vae.encode(start_image[:, :, :, :3]) + + m = model.clone() + to = m.model_options.setdefault("transformer_options", {}) + ar_cfg = to.setdefault("ar_config", {}) + ar_cfg["initial_latent"] = initial_latent + + lat_t = ((length - 1) // 4) + 1 + latent = torch.zeros( + [batch_size, 16, lat_t, height // 8, width // 8], + device=comfy.model_management.intermediate_device(), + ) + return io.NodeOutput(m, {"samples": latent}) + + +class ARVideoExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + EmptyARVideoLatent, + SamplerARVideo, + ARVideoI2V, + ] + + +async def comfy_entrypoint() -> ARVideoExtension: + return ARVideoExtension() diff --git a/ComfyUI/comfy_extras/nodes_attention_multiply.py b/ComfyUI/comfy_extras/nodes_attention_multiply.py new file mode 100644 index 0000000000000000000000000000000000000000..f4ee6a689edb2e073006959e3baf954c4df1787b --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_attention_multiply.py @@ -0,0 +1,151 @@ +from typing_extensions import override + +from comfy_api.latest import ComfyExtension, io + + +def attention_multiply(attn, model, q, k, v, out): + m = model.clone() + sd = model.model_state_dict() + + for key in sd: + if key.endswith("{}.to_q.bias".format(attn)) or key.endswith("{}.to_q.weight".format(attn)): + m.add_patches({key: (None,)}, 0.0, q) + if key.endswith("{}.to_k.bias".format(attn)) or key.endswith("{}.to_k.weight".format(attn)): + m.add_patches({key: (None,)}, 0.0, k) + if key.endswith("{}.to_v.bias".format(attn)) or key.endswith("{}.to_v.weight".format(attn)): + m.add_patches({key: (None,)}, 0.0, v) + if key.endswith("{}.to_out.0.bias".format(attn)) or key.endswith("{}.to_out.0.weight".format(attn)): + m.add_patches({key: (None,)}, 0.0, out) + + return m + + +class UNetSelfAttentionMultiply(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="UNetSelfAttentionMultiply", + category="experimental/attention_experiments", + inputs=[ + io.Model.Input("model"), + io.Float.Input("q", default=1.0, min=0.0, max=10.0, step=0.01, advanced=True), + io.Float.Input("k", default=1.0, min=0.0, max=10.0, step=0.01, advanced=True), + io.Float.Input("v", default=1.0, min=0.0, max=10.0, step=0.01, advanced=True), + io.Float.Input("out", default=1.0, min=0.0, max=10.0, step=0.01, advanced=True), + ], + outputs=[io.Model.Output()], + is_experimental=True, + ) + + @classmethod + def execute(cls, model, q, k, v, out) -> io.NodeOutput: + m = attention_multiply("attn1", model, q, k, v, out) + return io.NodeOutput(m) + + +class UNetCrossAttentionMultiply(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="UNetCrossAttentionMultiply", + category="experimental/attention_experiments", + inputs=[ + io.Model.Input("model"), + io.Float.Input("q", default=1.0, min=0.0, max=10.0, step=0.01, advanced=True), + io.Float.Input("k", default=1.0, min=0.0, max=10.0, step=0.01, advanced=True), + io.Float.Input("v", default=1.0, min=0.0, max=10.0, step=0.01, advanced=True), + io.Float.Input("out", default=1.0, min=0.0, max=10.0, step=0.01, advanced=True), + ], + outputs=[io.Model.Output()], + is_experimental=True, + ) + + @classmethod + def execute(cls, model, q, k, v, out) -> io.NodeOutput: + m = attention_multiply("attn2", model, q, k, v, out) + return io.NodeOutput(m) + + +class CLIPAttentionMultiply(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="CLIPAttentionMultiply", + search_aliases=["clip attention scale", "text encoder attention"], + category="experimental/attention_experiments", + inputs=[ + io.Clip.Input("clip"), + io.Float.Input("q", default=1.0, min=0.0, max=10.0, step=0.01, advanced=True), + io.Float.Input("k", default=1.0, min=0.0, max=10.0, step=0.01, advanced=True), + io.Float.Input("v", default=1.0, min=0.0, max=10.0, step=0.01, advanced=True), + io.Float.Input("out", default=1.0, min=0.0, max=10.0, step=0.01, advanced=True), + ], + outputs=[io.Clip.Output()], + is_experimental=True, + ) + + @classmethod + def execute(cls, clip, q, k, v, out) -> io.NodeOutput: + m = clip.clone() + sd = m.patcher.model_state_dict() + + for key in sd: + if key.endswith("self_attn.q_proj.weight") or key.endswith("self_attn.q_proj.bias"): + m.add_patches({key: (None,)}, 0.0, q) + if key.endswith("self_attn.k_proj.weight") or key.endswith("self_attn.k_proj.bias"): + m.add_patches({key: (None,)}, 0.0, k) + if key.endswith("self_attn.v_proj.weight") or key.endswith("self_attn.v_proj.bias"): + m.add_patches({key: (None,)}, 0.0, v) + if key.endswith("self_attn.out_proj.weight") or key.endswith("self_attn.out_proj.bias"): + m.add_patches({key: (None,)}, 0.0, out) + return io.NodeOutput(m) + + +class UNetTemporalAttentionMultiply(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="UNetTemporalAttentionMultiply", + category="experimental/attention_experiments", + inputs=[ + io.Model.Input("model"), + io.Float.Input("self_structural", default=1.0, min=0.0, max=10.0, step=0.01, advanced=True), + io.Float.Input("self_temporal", default=1.0, min=0.0, max=10.0, step=0.01, advanced=True), + io.Float.Input("cross_structural", default=1.0, min=0.0, max=10.0, step=0.01, advanced=True), + io.Float.Input("cross_temporal", default=1.0, min=0.0, max=10.0, step=0.01, advanced=True), + ], + outputs=[io.Model.Output()], + is_experimental=True, + ) + + @classmethod + def execute(cls, model, self_structural, self_temporal, cross_structural, cross_temporal) -> io.NodeOutput: + m = model.clone() + sd = model.model_state_dict() + + for k in sd: + if (k.endswith("attn1.to_out.0.bias") or k.endswith("attn1.to_out.0.weight")): + if '.time_stack.' in k: + m.add_patches({k: (None,)}, 0.0, self_temporal) + else: + m.add_patches({k: (None,)}, 0.0, self_structural) + elif (k.endswith("attn2.to_out.0.bias") or k.endswith("attn2.to_out.0.weight")): + if '.time_stack.' in k: + m.add_patches({k: (None,)}, 0.0, cross_temporal) + else: + m.add_patches({k: (None,)}, 0.0, cross_structural) + return io.NodeOutput(m) + + +class AttentionMultiplyExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + UNetSelfAttentionMultiply, + UNetCrossAttentionMultiply, + CLIPAttentionMultiply, + UNetTemporalAttentionMultiply, + ] + +async def comfy_entrypoint() -> AttentionMultiplyExtension: + return AttentionMultiplyExtension() diff --git a/ComfyUI/comfy_extras/nodes_audio.py b/ComfyUI/comfy_extras/nodes_audio.py new file mode 100644 index 0000000000000000000000000000000000000000..1c6e91fe4f7781baa27f35b79566dc22d9b8a9ac --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_audio.py @@ -0,0 +1,896 @@ +import av +import torchaudio +import torch +import comfy.model_management +import folder_paths +import os +import hashlib +import node_helpers +import logging +from typing_extensions import override +from comfy_api.latest import ComfyExtension, IO, UI + +class EmptyLatentAudio(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="EmptyLatentAudio", + display_name="Empty Latent Audio", + category="model/latent", + essentials_category="Audio", + inputs=[ + IO.Float.Input("seconds", default=47.6, min=1.0, max=1000.0, step=0.1), + IO.Int.Input( + "batch_size", default=1, min=1, max=4096, tooltip="The number of latent images in the batch.", + ), + ], + outputs=[IO.Latent.Output()], + ) + + @classmethod + def execute(cls, seconds, batch_size) -> IO.NodeOutput: + length = round((seconds * 44100 / 2048) / 2) * 2 + latent = torch.zeros([batch_size, 64, length], device=comfy.model_management.intermediate_device()) + return IO.NodeOutput({"samples": latent, "type": "audio", "downscale_ratio_temporal": 2048}) + + generate = execute # TODO: remove + + +class ConditioningStableAudio(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="ConditioningStableAudio", + category="model/conditioning/stable audio", + inputs=[ + IO.Conditioning.Input("positive"), + IO.Conditioning.Input("negative"), + IO.Float.Input("seconds_start", default=0.0, min=0.0, max=1000.0, step=0.1), + IO.Float.Input("seconds_total", default=47.0, min=0.0, max=1000.0, step=0.1), + ], + outputs=[ + IO.Conditioning.Output(display_name="positive"), + IO.Conditioning.Output(display_name="negative"), + ], + ) + + @classmethod + def execute(cls, positive, negative, seconds_start, seconds_total) -> IO.NodeOutput: + positive = node_helpers.conditioning_set_values(positive, {"seconds_start": seconds_start, "seconds_total": seconds_total}) + negative = node_helpers.conditioning_set_values(negative, {"seconds_start": seconds_start, "seconds_total": seconds_total}) + return IO.NodeOutput(positive, negative) + + append = execute # TODO: remove + + +class VAEEncodeAudio(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="VAEEncodeAudio", + search_aliases=["audio to latent"], + display_name="VAE Encode Audio", + category="model/latent", + inputs=[ + IO.Audio.Input("audio"), + IO.Vae.Input("vae"), + ], + outputs=[IO.Latent.Output()], + ) + + @classmethod + def execute(cls, vae, audio) -> IO.NodeOutput: + if audio is None: + raise ValueError("VAEEncodeAudio: input audio is None (source video may have no audio track).") + sample_rate = audio["sample_rate"] + vae_sample_rate = getattr(vae, "audio_sample_rate", 44100) + if vae_sample_rate != sample_rate: + waveform = torchaudio.functional.resample(audio["waveform"], sample_rate, vae_sample_rate) + else: + waveform = audio["waveform"] + + t = vae.encode(waveform.movedim(1, -1)) + return IO.NodeOutput({"samples": t}) + + encode = execute # TODO: remove + + +def vae_decode_audio(vae, samples, tile=None, overlap=None): + latent = samples["samples"] + if latent.is_nested: + latent = latent.unbind()[-1] + + if tile is not None: + audio = vae.decode_tiled(latent, tile_x=tile, tile_y=tile, overlap=overlap).movedim(-1, 1) + else: + audio = vae.decode(latent).movedim(-1, 1) + + std = torch.std(audio, dim=[1, 2], keepdim=True) * 5.0 + std[std < 1.0] = 1.0 + audio /= std + vae_sample_rate = getattr(vae, "audio_sample_rate_output", getattr(vae, "audio_sample_rate", 44100)) + return {"waveform": audio, "sample_rate": vae_sample_rate if "sample_rate" not in samples else samples["sample_rate"]} + + +class VAEDecodeAudio(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="VAEDecodeAudio", + search_aliases=["latent to audio"], + display_name="VAE Decode Audio", + category="model/latent", + inputs=[ + IO.Latent.Input("samples"), + IO.Vae.Input("vae"), + ], + outputs=[IO.Audio.Output()], + ) + + @classmethod + def execute(cls, vae, samples) -> IO.NodeOutput: + return IO.NodeOutput(vae_decode_audio(vae, samples)) + + decode = execute # TODO: remove + + +class VAEDecodeAudioTiled(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="VAEDecodeAudioTiled", + search_aliases=["latent to audio"], + display_name="VAE Decode Audio (Tiled)", + category="model/latent", + inputs=[ + IO.Latent.Input("samples"), + IO.Vae.Input("vae"), + IO.Int.Input("tile_size", default=512, min=32, max=8192, step=8), + IO.Int.Input("overlap", default=64, min=0, max=1024, step=8), + ], + outputs=[IO.Audio.Output()], + ) + + @classmethod + def execute(cls, vae, samples, tile_size, overlap) -> IO.NodeOutput: + return IO.NodeOutput(vae_decode_audio(vae, samples, tile_size, overlap)) + + +class SaveAudio(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="SaveAudio", + search_aliases=["export flac"], + display_name="Save Audio (FLAC) (DEPRECATED)", + category="audio", + essentials_category="Audio", + inputs=[ + IO.Audio.Input("audio"), + IO.String.Input("filename_prefix", default="audio/ComfyUI"), + ], + hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], + is_deprecated=True, + is_output_node=True, + outputs=[IO.Audio.Output("audio")] + ) + + @classmethod + def execute(cls, audio, filename_prefix="ComfyUI", format="flac") -> IO.NodeOutput: + if audio is None: + raise ValueError("SaveAudio: input audio is None (source video may have no audio track).") + return IO.NodeOutput( + audio, + ui=UI.AudioSaveHelper.get_save_audio_ui(audio, filename_prefix=filename_prefix, cls=cls, format=format) + ) + + +class SaveAudioMP3(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="SaveAudioMP3", + search_aliases=["export mp3"], + display_name="Save Audio (MP3) (DEPRECATED)", + category="audio", + essentials_category="Audio", + inputs=[ + IO.Audio.Input("audio"), + IO.String.Input("filename_prefix", default="audio/ComfyUI"), + IO.Combo.Input("quality", options=["V0", "128k", "320k"], default="V0"), + ], + hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], + is_deprecated=True, + is_output_node=True, + outputs=[IO.Audio.Output("audio")] + ) + + @classmethod + def execute(cls, audio, filename_prefix="ComfyUI", format="mp3", quality="128k") -> IO.NodeOutput: + if audio is None: + raise ValueError("SaveAudioMP3: input audio is None (source video may have no audio track).") + return IO.NodeOutput( + audio, + ui=UI.AudioSaveHelper.get_save_audio_ui( + audio, filename_prefix=filename_prefix, cls=cls, format=format, quality=quality + ) + ) + + +class SaveAudioOpus(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="SaveAudioOpus", + search_aliases=["export opus"], + display_name="Save Audio (Opus) (DEPRECATED)", + category="audio", + inputs=[ + IO.Audio.Input("audio"), + IO.String.Input("filename_prefix", default="audio/ComfyUI"), + IO.Combo.Input("quality", options=["64k", "96k", "128k", "192k", "320k"], default="128k"), + ], + hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], + is_deprecated=True, + is_output_node=True, + outputs=[IO.Audio.Output("audio")] + ) + + @classmethod + def execute(cls, audio, filename_prefix="ComfyUI", format="opus", quality="V3") -> IO.NodeOutput: + if audio is None: + raise ValueError("SaveAudioOpus: input audio is None (source video may have no audio track).") + return IO.NodeOutput( + audio, + ui=UI.AudioSaveHelper.get_save_audio_ui( + audio, filename_prefix=filename_prefix, cls=cls, format=format, quality=quality + ) + ) + + +class SaveAudioAdvanced(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="SaveAudioAdvanced", + search_aliases=["save audio", "export audio", "output audio", "write audio", "flac", "mp3", "opus"], + display_name="Save Audio (Advanced)", + description="Saves the input audio to your ComfyUI output directory.", + category="audio", + inputs=[ + IO.Audio.Input("audio", tooltip="The audio to save."), + IO.String.Input( + "filename_prefix", + default="audio/ComfyUI", + tooltip=("The prefix for the file to save. May include formatting tokens such as %date:yyyy-MM-dd%."), + ), + IO.DynamicCombo.Input( + "format", + options=[ + IO.DynamicCombo.Option("flac", []), + IO.DynamicCombo.Option("mp3", [ + IO.Combo.Input("quality", options=["V0", "128k", "320k"], default="V0"), + ]), + IO.DynamicCombo.Option("opus", [ + IO.Combo.Input("quality", options=["64k", "96k", "128k", "192k", "320k"], default="128k"), + ]), + ], + tooltip="The file format in which to save the audio.", + ), + ], + hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], + is_output_node=True, + outputs=[IO.Audio.Output("audio")], + ) + + @classmethod + def execute(cls, audio, filename_prefix: str, format: dict) -> IO.NodeOutput: + file_format = format.get("format", None) + quality = format.get("quality", None) + if quality: + ui=UI.AudioSaveHelper.get_save_audio_ui(audio, filename_prefix=filename_prefix, cls=cls, format=file_format, quality=quality) + else: + ui=UI.AudioSaveHelper.get_save_audio_ui(audio, filename_prefix=filename_prefix, cls=cls, format=file_format) + return IO.NodeOutput(audio, ui=ui) + + +class PreviewAudio(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="PreviewAudio", + search_aliases=["play audio"], + display_name="Preview Audio", + category="audio", + description="Preview the audio without saving it to the ComfyUI output directory.", + inputs=[ + IO.Audio.Input("audio"), + ], + hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], + is_output_node=True, + outputs=[IO.Audio.Output("audio")] + ) + + @classmethod + def execute(cls, audio) -> IO.NodeOutput: + if audio is None: + raise ValueError("PreviewAudio: input audio is None (source video may have no audio track).") + return IO.NodeOutput(audio, ui=UI.PreviewAudio(audio, cls=cls)) + + save_flac = execute # TODO: remove + + +def f32_pcm(wav: torch.Tensor) -> torch.Tensor: + """Convert audio to float 32 bits PCM format.""" + if wav.dtype.is_floating_point: + return wav + elif wav.dtype == torch.int16: + return wav.float() / (2 ** 15) + elif wav.dtype == torch.int32: + return wav.float() / (2 ** 31) + raise ValueError(f"Unsupported wav dtype: {wav.dtype}") + +def load(filepath: str) -> tuple[torch.Tensor, int]: + with av.open(filepath) as af: + if not af.streams.audio: + raise ValueError("No audio stream found in the file.") + + stream = af.streams.audio[0] + sr = stream.codec_context.sample_rate + n_channels = stream.channels + + frames = [] + length = 0 + for frame in af.decode(streams=stream.index): + buf = torch.from_numpy(frame.to_ndarray()) + if buf.shape[0] != n_channels: + buf = buf.view(-1, n_channels).t() + + frames.append(buf) + length += buf.shape[1] + + if not frames: + raise ValueError("No audio frames decoded.") + + wav = torch.cat(frames, dim=1) + wav = f32_pcm(wav) + return wav, sr + +class LoadAudio(IO.ComfyNode): + @classmethod + def define_schema(cls): + input_dir = folder_paths.get_input_directory() + os.makedirs(input_dir, exist_ok=True) + files = folder_paths.filter_files_content_types(os.listdir(input_dir), ["audio", "video"]) + return IO.Schema( + node_id="LoadAudio", + search_aliases=["import audio", "open audio", "audio file"], + display_name="Load Audio", + category="audio", + essentials_category="Audio", + inputs=[ + IO.Combo.Input("audio", upload=IO.UploadType.audio, options=sorted(files)), + ], + outputs=[IO.Audio.Output()], + ) + + @classmethod + def execute(cls, audio) -> IO.NodeOutput: + audio_path = folder_paths.get_annotated_filepath(audio) + waveform, sample_rate = load(audio_path) + audio = {"waveform": waveform.unsqueeze(0), "sample_rate": sample_rate} + return IO.NodeOutput(audio) + + @classmethod + def fingerprint_inputs(cls, audio): + image_path = folder_paths.get_annotated_filepath(audio) + m = hashlib.sha256() + with open(image_path, 'rb') as f: + m.update(f.read()) + return m.digest().hex() + + @classmethod + def validate_inputs(cls, audio): + if not folder_paths.exists_annotated_filepath(audio): + return "Invalid audio file: {}".format(audio) + return True + + load = execute # TODO: remove + + +class RecordAudio(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="RecordAudio", + search_aliases=["microphone input", "audio capture", "voice input"], + display_name="Record Audio", + category="audio", + inputs=[ + IO.Custom("AUDIO_RECORD").Input("audio"), + ], + outputs=[IO.Audio.Output()], + ) + + @classmethod + def execute(cls, audio) -> IO.NodeOutput: + audio_path = folder_paths.get_annotated_filepath(audio) + + waveform, sample_rate = load(audio_path) + audio = {"waveform": waveform.unsqueeze(0), "sample_rate": sample_rate} + return IO.NodeOutput(audio) + + load = execute # TODO: remove + + +class TrimAudioDuration(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="TrimAudioDuration", + search_aliases=["cut audio", "audio clip", "shorten audio"], + display_name="Trim Audio Duration", + description="Trim audio tensor into chosen time range.", + category="audio", + inputs=[ + IO.Audio.Input("audio"), + IO.Float.Input( + "start_index", + default=0.0, + min=-0xffffffffffffffff, + max=0xffffffffffffffff, + step=0.01, + tooltip="Start time in seconds, can be negative to count from the end (supports sub-seconds).", + ), + IO.Float.Input( + "duration", + default=60.0, + min=0.0, + step=0.01, + tooltip="Duration in seconds", + ), + ], + outputs=[IO.Audio.Output()], + ) + + @classmethod + def execute(cls, audio, start_index, duration) -> IO.NodeOutput: + if audio is None: + return IO.NodeOutput(None) + waveform = audio["waveform"] + sample_rate = audio["sample_rate"] + audio_length = waveform.shape[-1] + + if audio_length == 0: + return IO.NodeOutput(audio) + + if start_index < 0: + start_frame = audio_length + int(round(start_index * sample_rate)) + else: + start_frame = int(round(start_index * sample_rate)) + start_frame = max(0, min(start_frame, audio_length)) + + end_frame = start_frame + int(round(duration * sample_rate)) + end_frame = max(0, min(end_frame, audio_length)) + + if start_frame >= end_frame: + raise ValueError("TrimAudioDuration: Start time must be less than end time and be within the audio length.") + + return IO.NodeOutput({"waveform": waveform[..., start_frame:end_frame], "sample_rate": sample_rate}) + + trim = execute # TODO: remove + + +class SplitAudioChannels(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="SplitAudioChannels", + search_aliases=["stereo to mono"], + display_name="Split Audio Channels", + description="Separates the audio into left and right channels.", + category="audio", + inputs=[ + IO.Audio.Input("audio"), + ], + outputs=[ + IO.Audio.Output(display_name="left"), + IO.Audio.Output(display_name="right"), + ], + ) + + @classmethod + def execute(cls, audio) -> IO.NodeOutput: + if audio is None: + return IO.NodeOutput(None, None) + waveform = audio["waveform"] + sample_rate = audio["sample_rate"] + + if waveform.shape[1] != 2: + raise ValueError(f"AudioSplit: Input audio must be stereo (2 channels), got {waveform.shape[1]} channel(s).") + + left_channel = waveform[..., 0:1, :] + right_channel = waveform[..., 1:2, :] + + return IO.NodeOutput({"waveform": left_channel, "sample_rate": sample_rate}, {"waveform": right_channel, "sample_rate": sample_rate}) + + separate = execute # TODO: remove + +class JoinAudioChannels(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="JoinAudioChannels", + display_name="Join Audio Channels", + description="Joins left and right mono audio channels into a stereo audio.", + category="audio", + inputs=[ + IO.Audio.Input("audio_left"), + IO.Audio.Input("audio_right"), + ], + outputs=[ + IO.Audio.Output(display_name="audio"), + ], + ) + + @classmethod + def execute(cls, audio_left, audio_right) -> IO.NodeOutput: + if audio_left is None and audio_right is None: + return IO.NodeOutput(None) + if audio_left is None: + return IO.NodeOutput(audio_right) + if audio_right is None: + return IO.NodeOutput(audio_left) + waveform_left = audio_left["waveform"] + sample_rate_left = audio_left["sample_rate"] + waveform_right = audio_right["waveform"] + sample_rate_right = audio_right["sample_rate"] + + if waveform_left.shape[1] != 1 or waveform_right.shape[1] != 1: + raise ValueError("AudioJoin: Both input audios must be mono.") + + # Handle different sample rates by resampling to the higher rate + waveform_left, waveform_right, output_sample_rate = match_audio_sample_rates( + waveform_left, sample_rate_left, waveform_right, sample_rate_right + ) + + # Handle different lengths by trimming to the shorter length + length_left = waveform_left.shape[-1] + length_right = waveform_right.shape[-1] + + if length_left != length_right: + min_length = min(length_left, length_right) + if length_left > min_length: + logging.info(f"JoinAudioChannels: Trimming left channel from {length_left} to {min_length} samples.") + waveform_left = waveform_left[..., :min_length] + if length_right > min_length: + logging.info(f"JoinAudioChannels: Trimming right channel from {length_right} to {min_length} samples.") + waveform_right = waveform_right[..., :min_length] + + # Join the channels into stereo + left_channel = waveform_left[..., 0:1, :] + right_channel = waveform_right[..., 0:1, :] + stereo_waveform = torch.cat([left_channel, right_channel], dim=1) + + return IO.NodeOutput({"waveform": stereo_waveform, "sample_rate": output_sample_rate}) + + +def match_audio_sample_rates(waveform_1, sample_rate_1, waveform_2, sample_rate_2): + if sample_rate_1 != sample_rate_2: + if sample_rate_1 > sample_rate_2: + waveform_2 = torchaudio.functional.resample(waveform_2, sample_rate_2, sample_rate_1) + output_sample_rate = sample_rate_1 + logging.info(f"Resampling audio2 from {sample_rate_2}Hz to {sample_rate_1}Hz for merging.") + else: + waveform_1 = torchaudio.functional.resample(waveform_1, sample_rate_1, sample_rate_2) + output_sample_rate = sample_rate_2 + logging.info(f"Resampling audio1 from {sample_rate_1}Hz to {sample_rate_2}Hz for merging.") + else: + output_sample_rate = sample_rate_1 + return waveform_1, waveform_2, output_sample_rate + + +class AudioConcat(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="AudioConcat", + search_aliases=["join audio", "combine audio", "append audio"], + display_name="Concatenate Audio", + description="Concatenates the audio1 to audio2 in the specified direction.", + category="audio", + inputs=[ + IO.Audio.Input("audio1"), + IO.Audio.Input("audio2"), + IO.Combo.Input( + "direction", + options=['after', 'before'], + default="after", + tooltip="Whether to append audio2 after or before audio1.", + ) + ], + outputs=[IO.Audio.Output()], + ) + + @classmethod + def execute(cls, audio1, audio2, direction) -> IO.NodeOutput: + if audio1 is None and audio2 is None: + return IO.NodeOutput(None) + if audio1 is None: + return IO.NodeOutput(audio2) + if audio2 is None: + return IO.NodeOutput(audio1) + waveform_1 = audio1["waveform"] + waveform_2 = audio2["waveform"] + sample_rate_1 = audio1["sample_rate"] + sample_rate_2 = audio2["sample_rate"] + + if waveform_1.shape[1] == 1: + waveform_1 = waveform_1.repeat(1, 2, 1) + logging.info("AudioConcat: Converted mono audio1 to stereo by duplicating the channel.") + if waveform_2.shape[1] == 1: + waveform_2 = waveform_2.repeat(1, 2, 1) + logging.info("AudioConcat: Converted mono audio2 to stereo by duplicating the channel.") + + waveform_1, waveform_2, output_sample_rate = match_audio_sample_rates(waveform_1, sample_rate_1, waveform_2, sample_rate_2) + + if direction == 'after': + concatenated_audio = torch.cat((waveform_1, waveform_2), dim=2) + elif direction == 'before': + concatenated_audio = torch.cat((waveform_2, waveform_1), dim=2) + + return IO.NodeOutput({"waveform": concatenated_audio, "sample_rate": output_sample_rate}) + + concat = execute # TODO: remove + + +class AudioMerge(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="AudioMerge", + search_aliases=["mix audio", "overlay audio", "layer audio"], + display_name="Merge Audio", + description="Combine two audio tracks by overlaying their waveforms.", + category="audio", + inputs=[ + IO.Audio.Input("audio1"), + IO.Audio.Input("audio2"), + IO.Combo.Input( + "merge_method", + options=["add", "mean", "subtract", "multiply"], + tooltip="The method used to combine the audio waveforms.", + ) + ], + outputs=[IO.Audio.Output()], + ) + + @classmethod + def execute(cls, audio1, audio2, merge_method) -> IO.NodeOutput: + if audio1 is None and audio2 is None: + return IO.NodeOutput(None) + if audio1 is None: + return IO.NodeOutput(audio2) + if audio2 is None: + return IO.NodeOutput(audio1) + waveform_1 = audio1["waveform"] + waveform_2 = audio2["waveform"] + sample_rate_1 = audio1["sample_rate"] + sample_rate_2 = audio2["sample_rate"] + + waveform_1, waveform_2, output_sample_rate = match_audio_sample_rates(waveform_1, sample_rate_1, waveform_2, sample_rate_2) + + length_1 = waveform_1.shape[-1] + length_2 = waveform_2.shape[-1] + + if length_1 == 0 or length_2 == 0: + return IO.NodeOutput({"waveform": waveform_1, "sample_rate": output_sample_rate}) + + if length_2 > length_1: + logging.info(f"AudioMerge: Trimming audio2 from {length_2} to {length_1} samples to match audio1 length.") + waveform_2 = waveform_2[..., :length_1] + elif length_2 < length_1: + logging.info(f"AudioMerge: Padding audio2 from {length_2} to {length_1} samples to match audio1 length.") + pad_shape = list(waveform_2.shape) + pad_shape[-1] = length_1 - length_2 + pad_tensor = torch.zeros(pad_shape, dtype=waveform_2.dtype, device=waveform_2.device) + waveform_2 = torch.cat((waveform_2, pad_tensor), dim=-1) + + if merge_method == "add": + waveform = waveform_1 + waveform_2 + elif merge_method == "subtract": + waveform = waveform_1 - waveform_2 + elif merge_method == "multiply": + waveform = waveform_1 * waveform_2 + elif merge_method == "mean": + waveform = (waveform_1 + waveform_2) / 2 + + max_val = waveform.abs().max() + if max_val > 1.0: + waveform = waveform / max_val + + return IO.NodeOutput({"waveform": waveform, "sample_rate": output_sample_rate}) + + merge = execute # TODO: remove + + +class AudioAdjustVolume(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="AudioAdjustVolume", + search_aliases=["audio gain", "loudness", "audio level"], + display_name="Adjust Audio Volume", + category="audio", + description="Adjust the volume of the audio by a specified amount in decibels (dB).", + inputs=[ + IO.Audio.Input("audio"), + IO.Int.Input( + "volume", + default=1, + min=-100, + max=100, + tooltip="Volume adjustment in decibels (dB). 0 = no change, +6 = double, -6 = half, etc", + ) + ], + outputs=[IO.Audio.Output()], + ) + + @classmethod + def execute(cls, audio, volume) -> IO.NodeOutput: + if audio is None: + return IO.NodeOutput(None) + if volume == 0: + return IO.NodeOutput(audio) + waveform = audio["waveform"] + sample_rate = audio["sample_rate"] + + gain = 10 ** (volume / 20) + waveform = waveform * gain + + return IO.NodeOutput({"waveform": waveform, "sample_rate": sample_rate}) + + adjust_volume = execute # TODO: remove + + +class EmptyAudio(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="EmptyAudio", + search_aliases=["blank audio"], + display_name="Empty Audio", + category="audio", + inputs=[ + IO.Float.Input( + "duration", + default=60.0, + min=0.0, + max=0xffffffffffffffff, + step=0.01, + tooltip="Duration of the empty audio clip in seconds", + ), + IO.Int.Input( + "sample_rate", + default=44100, + tooltip="Sample rate of the empty audio clip.", + min=1, + max=192000, + advanced=True, + ), + IO.Int.Input( + "channels", + default=2, + min=1, + max=2, + tooltip="Number of audio channels (1 for mono, 2 for stereo).", + advanced=True, + ), + ], + outputs=[IO.Audio.Output()], + ) + + @classmethod + def execute(cls, duration, sample_rate, channels) -> IO.NodeOutput: + num_samples = int(round(duration * sample_rate)) + waveform = torch.zeros((1, channels, num_samples), dtype=torch.float32) + return IO.NodeOutput({"waveform": waveform, "sample_rate": sample_rate}) + + create_empty_audio = execute # TODO: remove + + +class AudioEqualizer3Band(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="AudioEqualizer3Band", + search_aliases=["eq", "bass boost", "treble boost", "equalizer"], + display_name="Audio Equalizer (3-Band)", + category="audio", + is_experimental=True, + inputs=[ + IO.Audio.Input("audio"), + IO.Float.Input("low_gain_dB", default=0.0, min=-24.0, max=24.0, step=0.1, tooltip="Gain for Low frequencies (Bass)"), + IO.Int.Input("low_freq", default=100, min=20, max=500, tooltip="Cutoff frequency for Low shelf"), + IO.Float.Input("mid_gain_dB", default=0.0, min=-24.0, max=24.0, step=0.1, tooltip="Gain for Mid frequencies"), + IO.Int.Input("mid_freq", default=1000, min=200, max=4000, tooltip="Center frequency for Mids"), + IO.Float.Input("mid_q", default=0.707, min=0.1, max=10.0, step=0.1, tooltip="Q factor (bandwidth) for Mids"), + IO.Float.Input("high_gain_dB", default=0.0, min=-24.0, max=24.0, step=0.1, tooltip="Gain for High frequencies (Treble)"), + IO.Int.Input("high_freq", default=5000, min=1000, max=15000, tooltip="Cutoff frequency for High shelf"), + ], + outputs=[IO.Audio.Output()], + ) + + @classmethod + def execute(cls, audio, low_gain_dB, low_freq, mid_gain_dB, mid_freq, mid_q, high_gain_dB, high_freq) -> IO.NodeOutput: + if audio is None: + return IO.NodeOutput(None) + waveform = audio["waveform"] + sample_rate = audio["sample_rate"] + + if waveform.shape[-1] == 0: + return IO.NodeOutput(audio) + + eq_waveform = waveform.clone() + + # 1. Apply Low Shelf (Bass) + if low_gain_dB != 0: + eq_waveform = torchaudio.functional.bass_biquad( + eq_waveform, + sample_rate, + gain=low_gain_dB, + central_freq=float(low_freq), + Q=0.707 + ) + + # 2. Apply Peaking EQ (Mids) + if mid_gain_dB != 0: + eq_waveform = torchaudio.functional.equalizer_biquad( + eq_waveform, + sample_rate, + center_freq=float(mid_freq), + gain=mid_gain_dB, + Q=mid_q + ) + + # 3. Apply High Shelf (Treble) + if high_gain_dB != 0: + eq_waveform = torchaudio.functional.treble_biquad( + eq_waveform, + sample_rate, + gain=high_gain_dB, + central_freq=float(high_freq), + Q=0.707 + ) + + return IO.NodeOutput({"waveform": eq_waveform, "sample_rate": sample_rate}) + + +class AudioExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[IO.ComfyNode]]: + return [ + EmptyLatentAudio, + VAEEncodeAudio, + VAEDecodeAudio, + VAEDecodeAudioTiled, + SaveAudio, + SaveAudioMP3, + SaveAudioOpus, + SaveAudioAdvanced, + LoadAudio, + PreviewAudio, + ConditioningStableAudio, + RecordAudio, + TrimAudioDuration, + SplitAudioChannels, + JoinAudioChannels, + AudioConcat, + AudioMerge, + AudioAdjustVolume, + EmptyAudio, + AudioEqualizer3Band, + ] + +async def comfy_entrypoint() -> AudioExtension: + return AudioExtension() diff --git a/ComfyUI/comfy_extras/nodes_audio_encoder.py b/ComfyUI/comfy_extras/nodes_audio_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..2ae30d321b937d4f2ad9a678fbca341426882ec6 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_audio_encoder.py @@ -0,0 +1,63 @@ +import folder_paths +import comfy.audio_encoders.audio_encoders +import comfy.utils +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io + + +class AudioEncoderLoader(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="AudioEncoderLoader", + display_name="Load Audio Encoder", + category="model/loaders", + inputs=[ + io.Combo.Input( + "audio_encoder_name", + options=folder_paths.get_filename_list("audio_encoders"), + ), + ], + outputs=[io.AudioEncoder.Output()], + ) + + @classmethod + def execute(cls, audio_encoder_name) -> io.NodeOutput: + audio_encoder_name = folder_paths.get_full_path_or_raise("audio_encoders", audio_encoder_name) + sd = comfy.utils.load_torch_file(audio_encoder_name, safe_load=True) + audio_encoder = comfy.audio_encoders.audio_encoders.load_audio_encoder_from_sd(sd) + if audio_encoder is None: + raise RuntimeError("ERROR: audio encoder file is invalid and does not contain a valid model.") + return io.NodeOutput(audio_encoder) + + +class AudioEncoderEncode(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="AudioEncoderEncode", + category="model/conditioning", + inputs=[ + io.AudioEncoder.Input("audio_encoder"), + io.Audio.Input("audio"), + ], + outputs=[io.AudioEncoderOutput.Output()], + ) + + @classmethod + def execute(cls, audio_encoder, audio) -> io.NodeOutput: + output = audio_encoder.encode_audio(audio["waveform"], audio["sample_rate"]) + return io.NodeOutput(output) + + +class AudioEncoder(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + AudioEncoderLoader, + AudioEncoderEncode, + ] + + +async def comfy_entrypoint() -> AudioEncoder: + return AudioEncoder() diff --git a/ComfyUI/comfy_extras/nodes_bernini.py b/ComfyUI/comfy_extras/nodes_bernini.py new file mode 100644 index 0000000000000000000000000000000000000000..0537e08063bee203133d0315eb766a5524f5c70a --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_bernini.py @@ -0,0 +1,108 @@ +import torch +from typing_extensions import override + +import comfy.model_management +import comfy.utils +import node_helpers +from comfy_api.latest import ComfyExtension, io + + +def _resize_long_edge(image, max_size, stride=16): + """Resize (preserve aspect) so the long edge <= max_size, then snap each side to `stride`""" + h, w = image.shape[1], image.shape[2] + scale = min(max_size / max(h, w), 1.0) + nh = max(stride, round(h * scale / stride) * stride) + nw = max(stride, round(w * scale / stride) * stride) + return comfy.utils.common_upscale(image[:, :, :, :3].movedim(-1, 1), nw, nh, "area", "disabled").movedim(1, -1) + + +class BerniniConditioning(io.ComfyNode): + """Bernini in-context conditioning for a Wan2.2-A14B model. + + Attaches the VAE-encoded source video / reference images to the conditioning + source video first, then each reference image + + The task is inferred from which inputs are connected: + (nothing) -> t2v (text-to-video) + source_video -> v2v (video-to-video) + source_video + ref_images -> rv2v (reference-guided video editing) + ref_images only -> r2v (reference-to-video) + source_video + ref_video -> ads2v (insert image/video into video) + + source_video is the edit base / canvas (resized to width x height). + reference_video is moving content to composite in. + Streams are ordered source_video, reference_video, then reference_images -> source_id (1, 2, 3, ...). + """ + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="BerniniConditioning", + display_name="Bernini Conditioning", + category="model/conditioning/bernini", + description="Conditioning node for Bernini in-context video/image conditioning. It can be used for the following tasks: t2v (text-to-video), v2v (video-to-video), rv2v (reference-guided video editing), r2v (reference-to-video), ads2v (insert image/video into video)." + "Reference images injected as in-context tokens (r2v, rv2v) are encoded independently at their own native aspect ratio (long edge capped at ref_max_size)", + inputs=[ + io.Conditioning.Input("positive"), + io.Conditioning.Input("negative"), + io.Vae.Input("vae"), + io.Int.Input("width", default=832, min=16, max=8192, step=16), + io.Int.Input("height", default=480, min=16, max=8192, step=16), + io.Int.Input("length", default=81, min=1, max=8192, step=4), + io.Int.Input("batch_size", default=1, min=1, max=4096), + io.Image.Input("source_video", optional=True, tooltip=("Source video to edit or restyle (v2v, rv2v). Resized to width/height and trimmed to length.")), + io.Image.Input("reference_video", optional=True, tooltip=("Video to insert into the source video (ads2v).")), + io.Autogrow.Input("reference_images", optional=True, + template=io.Autogrow.TemplatePrefix( + input=io.Image.Input("reference_image", tooltip=("Reference image injected as an in-context token (r2v, rv2v).")), + prefix="reference_image_", min=0, max=8)), + io.Int.Input("ref_max_size", default=848, min=16, max=8192, step=16, optional=True, tooltip=( + "Max size for the long edge of reference_video and reference_images. Resized with preserved aspect ratio and snapped to 16px.")), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + io.Latent.Output(display_name="latent"), + ], + ) + + @classmethod + def execute(cls, positive, negative, vae, width, height, length, batch_size, source_video=None, reference_video=None, reference_images=None, ref_max_size=848) -> io.NodeOutput: + latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device()) + + # source_video (1), reference_video (2), reference_images (3, 4, ...). + context = [] + if source_video is not None: + vid = comfy.utils.common_upscale(source_video[:length, :, :, :3].movedim(-1, 1), width, height, "area", "center").movedim(1, -1) + context.append(vae.encode(vid[:, :, :, :3])) + + if reference_video is not None: + ref_vid = _resize_long_edge(reference_video[:length], ref_max_size) # moving content, native aspect + context.append(vae.encode(ref_vid[:, :, :, :3])) + + # reference_images is an autogrow dict {reference_image_0: IMAGE, ...}; each slot is a + # separate stream at its own native aspect (a multi-image batch in one slot -> one stream per frame). + if reference_images: + for name in sorted(reference_images): + imgs = reference_images[name] + if imgs is None: + continue + for i in range(imgs.shape[0]): + img = _resize_long_edge(imgs[i:i + 1], ref_max_size) # native aspect per ref + context.append(vae.encode(img[:, :, :, :3])) + + if context: + positive = node_helpers.conditioning_set_values(positive, {"context_latents": context}) + negative = node_helpers.conditioning_set_values(negative, {"context_latents": context}) + + return io.NodeOutput(positive, negative, {"samples": latent}) + + +class BerniniExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [BerniniConditioning,] + + +async def comfy_entrypoint() -> BerniniExtension: + return BerniniExtension() diff --git a/ComfyUI/comfy_extras/nodes_bg_removal.py b/ComfyUI/comfy_extras/nodes_bg_removal.py new file mode 100644 index 0000000000000000000000000000000000000000..c7b33a821720c5271354781d6c076b2b281d9d1e --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_bg_removal.py @@ -0,0 +1,61 @@ +import folder_paths +from typing_extensions import override +from comfy_api.latest import ComfyExtension, IO +from comfy.bg_removal_model import load + + +class LoadBackgroundRemovalModel(IO.ComfyNode): + @classmethod + def define_schema(cls): + files = folder_paths.get_filename_list("background_removal") + return IO.Schema( + node_id="LoadBackgroundRemovalModel", + display_name="Load Background Removal Model", + category="model/loaders", + inputs=[ + IO.Combo.Input("bg_removal_name", options=sorted(files), tooltip="The model used to remove backgrounds from images"), + ], + outputs=[ + IO.BackgroundRemoval.Output("bg_model") + ] + ) + @classmethod + def execute(cls, bg_removal_name): + path = folder_paths.get_full_path_or_raise("background_removal", bg_removal_name) + bg = load(path) + if bg is None: + raise RuntimeError("ERROR: background model file is invalid and does not contain a valid background removal model.") + return IO.NodeOutput(bg) + +class RemoveBackground(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="RemoveBackground", + display_name="Remove Background", + category="image/background removal", + description="Generates a foreground mask to remove the background from an image using a background removal model.", + inputs=[ + IO.BackgroundRemoval.Input("bg_removal_model", tooltip="Background removal model used to generate the mask"), + IO.Image.Input("image", tooltip="Input image to remove the background from") + ], + outputs=[ + IO.Mask.Output("mask", tooltip="Generated foreground mask") + ] + ) + @classmethod + def execute(cls, bg_removal_model, image): + mask = bg_removal_model.encode_image(image) + return IO.NodeOutput(mask) + +class BackgroundRemovalExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[IO.ComfyNode]]: + return [ + LoadBackgroundRemovalModel, + RemoveBackground + ] + + +async def comfy_entrypoint() -> BackgroundRemovalExtension: + return BackgroundRemovalExtension() diff --git a/ComfyUI/comfy_extras/nodes_boogu.py b/ComfyUI/comfy_extras/nodes_boogu.py new file mode 100644 index 0000000000000000000000000000000000000000..f3951c29039d0ec873a7158d12bdf3d1d539636b --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_boogu.py @@ -0,0 +1,97 @@ +import math + +import node_helpers +import comfy.utils +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io + + +class TextEncodeBooguEdit(io.ComfyNode): + """Boogu-Image Edit conditioning. + + The edit image is used twice, matching the reference pipeline: + - Qwen3-VL vision tokens (instruction understanding) -> positive only + - VAE reference latent (image identity) -> positive and negative + The ref latent is in both conds so it cancels under CFG (identity preserved); + the vision tokens are only in the positive so CFG amplifies the instruction. + The tokenizer selects the right system prompt automatically (image -> TI2I, + empty negative -> DROP), so no template plumbing is needed here. + """ + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="TextEncodeBooguEdit", + category="model/conditioning/boogu", + inputs=[ + io.Clip.Input("clip"), + io.String.Input("prompt", multiline=True, dynamic_prompts=True), + io.String.Input("negative_prompt", multiline=True, dynamic_prompts=True, advanced=True), + io.Vae.Input("vae"), + io.Autogrow.Input( + "images", + template=io.Autogrow.TemplateNames( + io.Image.Input("image"), + names=[f"image_{i}" for i in range(1, 17)], + min=0, + ), + tooltip="Reference image(s) to edit. Boogu focuses on one reference per sample; more are allowed.", + ), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + ], + ) + + @classmethod + def execute(cls, clip, prompt, negative_prompt, vae=None, images: io.Autogrow.Type = None) -> io.NodeOutput: + ref_latents = [] + images_vl = [] + + images = images or {} + for name in sorted(images, key=lambda n: int(n.rsplit("_", 1)[-1])): + image = images[name] + if image is None: + continue + samples = image.movedim(-1, 1) + + # Vision tower input: the reference caps the VLM image at 384x384 + # (max_vlm_input_pil_pixels in pipeline_boogu.py). + total = int(384 * 384) + scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2])) + width = round(samples.shape[3] * scale_by) + height = round(samples.shape[2] * scale_by) + s = comfy.utils.common_upscale(samples, width, height, "area", "disabled") + images_vl.append(s.movedim(1, -1)[:, :, :, :3]) + + # Reference latent: align to 16 px (VAE /8 * patch_size 2). + if vae is not None: + total = int(1024 * 1024) + scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2])) + width = round(samples.shape[3] * scale_by / 16.0) * 16 + height = round(samples.shape[2] * scale_by / 16.0) * 16 + s = comfy.utils.common_upscale(samples, width, height, "area", "disabled") + ref_latents.append(vae.encode(s.movedim(1, -1)[:, :, :, :3])) + + # positive: instruction + vision tokens; negative: empty (no vision). Ref latent on both. + positive = clip.encode_from_tokens_scheduled(clip.tokenize(prompt, images=images_vl)) + negative = clip.encode_from_tokens_scheduled(clip.tokenize(negative_prompt)) + + if len(ref_latents) > 0: + positive = node_helpers.conditioning_set_values(positive, {"reference_latents": ref_latents}, append=True) + negative = node_helpers.conditioning_set_values(negative, {"reference_latents": ref_latents}, append=True) + + return io.NodeOutput(positive, negative) + + +class BooguExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + TextEncodeBooguEdit, + ] + + +async def comfy_entrypoint() -> BooguExtension: + return BooguExtension() diff --git a/ComfyUI/comfy_extras/nodes_bounding_boxes.py b/ComfyUI/comfy_extras/nodes_bounding_boxes.py new file mode 100644 index 0000000000000000000000000000000000000000..de3709b91009c9068bea60e2762db851f3b02584 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_bounding_boxes.py @@ -0,0 +1,379 @@ +import json + +import numpy as np +import torch +from PIL import Image, ImageDraw, ImageEnhance, ImageFont +from typing_extensions import override + +from comfy_api.latest import ComfyExtension, io +from comfy_extras.color_util import hex_to_rgb, normalize_palette, readable_color + +_PREVIEW_LONG_EDGE = 1024 +_PREVIEW_DIM = 0.25 + + +def pixels_to_fractions(box: dict, width: int, height: int) -> dict: + w = width or 1 + h = height or 1 + return { + "x": box.get("x", 0) / w, + "y": box.get("y", 0) / h, + "w": box.get("width", 0) / w, + "h": box.get("height", 0) / h, + } + + +def fractions_to_pixels(box: dict, width: int, height: int) -> dict: + x, y = box.get("x", 0.0), box.get("y", 0.0) + w, h = box.get("w", 0.0), box.get("h", 0.0) + if w < 0: + x, w = x + w, -w + if h < 0: + y, h = y + h, -h + return { + "x": round(x * width), + "y": round(y * height), + "width": round(w * width), + "height": round(h * height), + } + + +def fractions_to_bbox_frame(boxes: list, width: int, height: int) -> list: + pixels = [ + fractions_to_pixels(box, width, height) + for box in boxes + if isinstance(box, dict) + ] + return [pixels] if pixels else [] + + +def _font(size: int): + try: + return ImageFont.load_default(size) + except Exception: + return ImageFont.load_default() + + +def _wrap(draw, text: str, font, max_w: float) -> list[str]: + lines = [] + for para in text.split("\n"): + line = "" + for word in para.split(): + test = word if not line else line + " " + word + if line and draw.textlength(test, font=font) > max_w: + lines.append(line) + line = word + else: + line = test + lines.append(line) + return lines + + +def _bg_from_image(image) -> Image.Image | None: + if image is None: + return None + try: + arr = (image[0].detach().cpu().numpy() * 255).clip(0, 255).astype(np.uint8) + return Image.fromarray(arr) + except Exception: + return None + + +def render_preview(regions, width, height, bg=None): + if bg is not None: + iw, ih = bg.size + long_edge = max(iw, ih) or 1 + scale = min(1.0, _PREVIEW_LONG_EDGE / long_edge) + rw, rh = max(1, round(iw * scale)), max(1, round(ih * scale)) + base = bg.convert("RGB").resize((rw, rh), Image.LANCZOS) + base = ImageEnhance.Brightness(base).enhance(_PREVIEW_DIM) + img = base.convert("RGBA") + else: + long_edge = max(width, height) or 1 + scale = min(1.0, _PREVIEW_LONG_EDGE / long_edge) + rw, rh = max(1, round(width * scale)), max(1, round(height * scale)) + grey = round(_PREVIEW_DIM * 128) + img = Image.new("RGBA", (rw, rh), (grey, grey, grey, 255)) + + overlay = Image.new("RGBA", (rw, rh), (0, 0, 0, 0)) + draw = ImageDraw.Draw(overlay) + fs = max(10, round(rh / 64)) + font = _font(fs) + tag_font = _font(max(9, fs - 2)) + line_h = fs + 2 + + for i, region in enumerate(regions): + if not isinstance(region, dict): + continue + palette = [c for c in (region.get("palette") or []) if c] + r, g, b = hex_to_rgb(palette[0]) if palette else (140, 140, 140) + x1 = max(0, min(rw, round(region.get("x", 0) * rw))) + y1 = max(0, min(rh, round(region.get("y", 0) * rh))) + x2 = max(0, min(rw, round((region.get("x", 0) + region.get("w", 0)) * rw))) + y2 = max(0, min(rh, round((region.get("y", 0) + region.get("h", 0)) * rh))) + if x2 < x1: + x1, x2 = x2, x1 + if y2 < y1: + y1, y2 = y2, y1 + + draw.rectangle([x1, y1, x2, y2], outline=(r, g, b, 255), width=2) + + swatches = palette[:5] + if swatches and (x2 - x1) > 2: + sh = max(5, fs // 2) + seg = (x2 - x1) / len(swatches) + for p, hexc in enumerate(swatches): + sx = x1 + round(p * seg) + draw.rectangle([sx, y1, x1 + round((p + 1) * seg), y1 + sh], fill=hex_to_rgb(hexc)) + + etype = "text" if region.get("type") == "text" else "obj" + tag = str(i + 1).zfill(2) + tw = draw.textlength(tag, font=tag_font) + draw.rectangle([x1, y1, x1 + tw + 6, y1 + fs + 2], fill=(r, g, b, 255)) + tag_fill = (0, 0, 0, 255) if (0.299 * r + 0.587 * g + 0.114 * b) > 140 else (255, 255, 255, 255) + draw.text((x1 + 3, y1 + 1), tag, fill=tag_fill, font=tag_font) + + body = region.get("desc", "") or "" + if etype == "text" and region.get("text"): + body = '"%s"%s' % (region["text"], " — " + body if body else "") + if body and (x2 - x1) > 8: + ty = y1 + fs + 5 + for line in _wrap(draw, body, font, x2 - x1 - 8): + if ty > y2: + break + draw.text((x1 + 4, ty), line, fill=readable_color((r, g, b)) + (255,), font=font) + ty += line_h + + composed = Image.alpha_composite(img, overlay).convert("RGB") + arr = np.asarray(composed, dtype=np.float32) / 255.0 + return torch.from_numpy(arr).unsqueeze(0) + + +def boxes_to_regions(boxes, width: int, height: int) -> list: + regions: list = [] + if not isinstance(boxes, list): + return regions + for box in boxes: + if not isinstance(box, dict): + continue + meta = box.get("metadata") + meta = meta if isinstance(meta, dict) else {} + regions.append({ + **pixels_to_fractions(box, width, height), + "type": meta.get("type", "obj"), + "text": meta.get("text", ""), + "desc": meta.get("desc", ""), + "palette": meta.get("palette", []), + }) + return regions + + +def normalize_incoming_boxes(bboxes) -> list: + if isinstance(bboxes, dict): + frame = [bboxes] + elif not isinstance(bboxes, list) or not bboxes: + frame = [] + elif isinstance(bboxes[0], dict): + frame = bboxes + else: + frame = bboxes[0] if isinstance(bboxes[0], list) else [] + boxes = [] + for box in frame: + if not isinstance(box, dict): + continue + norm = { + "x": box.get("x", 0), + "y": box.get("y", 0), + "width": box.get("width", 0), + "height": box.get("height", 0), + } + meta = box.get("metadata") + if isinstance(meta, dict): + norm["metadata"] = meta + boxes.append(norm) + return boxes + + +def _looks_like_element(box: dict) -> bool: + bbox = box.get("bbox") + return isinstance(bbox, (list, tuple)) and len(bbox) == 4 + + +def _looks_like_bbox(box: dict) -> bool: + return all(key in box for key in ("x", "y", "width", "height")) + + +def elements_to_boxes(elements: list, width: int, height: int) -> list: + boxes = [] + for element in elements: + if not isinstance(element, dict): + continue + bbox = element.get("bbox") + if not (isinstance(bbox, (list, tuple)) and len(bbox) == 4): + raise ValueError("bboxes element is missing a valid 'bbox' [ymin, xmin, ymax, xmax]") + try: + ymin, xmin, ymax, xmax = (float(v) / 1000.0 for v in bbox) + except (TypeError, ValueError): + raise ValueError("bboxes element 'bbox' must contain four numbers") + etype = "text" if element.get("type") == "text" else "obj" + boxes.append({ + "x": round(min(xmin, xmax) * width), + "y": round(min(ymin, ymax) * height), + "width": round(abs(xmax - xmin) * width), + "height": round(abs(ymax - ymin) * height), + "metadata": { + "type": etype, + "text": element.get("text", "") if etype == "text" else "", + "desc": element.get("desc", ""), + "palette": element.get("color_palette", []) or [], + }, + }) + return boxes + + +def boxes_from_input(data, width: int, height: int) -> list: + if data is None: + return [] + if isinstance(data, str): + text = data.strip() + if not text: + return [] + try: + data = json.loads(text) + except (ValueError, TypeError) as exc: + raise ValueError(f"bboxes string input is not valid JSON: {exc}") from exc + if isinstance(data, dict): + if _looks_like_element(data): + return elements_to_boxes([data], width, height) + if _looks_like_bbox(data): + return normalize_incoming_boxes(data) + raise ValueError( + "bboxes dict must be a bounding box (x, y, width, height) or an element (with a 'bbox')" + ) + if not isinstance(data, list): + raise ValueError( + "bboxes input must be bounding boxes, elements, or a JSON string, " + f"got {type(data).__name__}" + ) + if not data: + return [] + first = data[0] + if isinstance(first, list): + return normalize_incoming_boxes(data) + if isinstance(first, dict): + if _looks_like_element(first): + return elements_to_boxes(data, width, height) + if _looks_like_bbox(first): + return normalize_incoming_boxes(data) + raise ValueError( + "bboxes items must be bounding boxes (x, y, width, height) or elements (with a 'bbox')" + ) + raise ValueError( + f"bboxes list must contain bounding boxes or elements, got {type(first).__name__}" + ) + + +def _norm_bbox(region: dict) -> list[int]: + def grid(value: float) -> int: + return max(0, min(1000, round(value * 1000))) + + x, y = region.get("x", 0.0), region.get("y", 0.0) + w, h = region.get("w", 0.0), region.get("h", 0.0) + ymin, xmin, ymax, xmax = grid(y), grid(x), grid(y + h), grid(x + w) + if ymin > ymax: + ymin, ymax = ymax, ymin + if xmin > xmax: + xmin, xmax = xmax, xmin + return [ymin, xmin, ymax, xmax] + + +def build_elements(regions: list) -> list: + elements = [] + for region in regions: + if not isinstance(region, dict): + continue + etype = "text" if region.get("type") == "text" else "obj" + element = {"type": etype} + element["bbox"] = _norm_bbox(region) + if etype == "text": + element["text"] = region.get("text", "") + element["desc"] = region.get("desc", "") + palette = normalize_palette(region.get("palette", [])) + if palette: + element["color_palette"] = palette[:5] + elements.append(element) + return elements + + +class CreateBoundingBoxes(io.ComfyNode): + @classmethod + def define_schema(cls): + editor_state = io.BoundingBoxes.Input( + "editor_state", + socketless=False, + tooltip="Draw bounding boxes and set each box type, text, description, color palette. Start with background element first and foreground last.", + ) + return io.Schema( + node_id="CreateBoundingBoxes", + display_name="Create Bounding Boxes", + category="utilities", + description="Draw bounding boxes in a canvas. Outputs Ideogram prompt elements, pixel-space bounding boxes, and a preview image.", + inputs=[ + io.Image.Input( + "background", + optional=True, + tooltip="Optional image used as background in the canvas and preview.", + ), + io.MultiType.Input( + "bboxes", + [io.BoundingBox, io.Array, io.String], + optional=True, + tooltip="Bounding boxes, elements, or a JSON string to initialize the canvas. A new upstream value initializes the canvas; edits made on the canvas take priority and are kept until the upstream value changes again.", + ), + io.Int.Input("width", default=1024, min=64, max=16384, step=16, + tooltip="Width of the canvas and the pixel grid for the bounding boxes."), + io.Int.Input("height", default=1024, min=64, max=16384, step=16, + tooltip="Height of the canvas and the pixel grid for the bounding boxes."), + editor_state, + io.BoundingBoxes.Input( + "last_incoming", + optional=True, + tooltip="Internal state managed by the canvas: the upstream bboxes value that last initialized it. Leave empty to re-initialize the canvas from the bboxes input on the next run.", + ), + ], + outputs=[ + io.Image.Output(display_name="preview"), + io.BoundingBox.Output(display_name="bboxes"), + io.Array.Output(display_name="elements"), + ], + is_output_node=True, + is_experimental=True, + ) + + @classmethod + def execute(cls, width, height, editor_state=None, last_incoming=None, background=None, bboxes=None) -> io.NodeOutput: + incoming = boxes_from_input(bboxes, width, height) + applied = last_incoming if isinstance(last_incoming, list) else [] + upstream_changed = bool(incoming) and incoming != applied + source = incoming if upstream_changed else (editor_state or []) + regions = boxes_to_regions(source, width, height) + preview = render_preview(regions, width, height, _bg_from_image(background)) + ui = {"dims": [width, height]} + if incoming: + ui["input_bboxes"] = incoming + return io.NodeOutput( + preview, + fractions_to_bbox_frame(regions, width, height), + build_elements(regions), + ui=ui, + ) + + +class BoundingBoxesExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [CreateBoundingBoxes] + + +async def comfy_entrypoint() -> BoundingBoxesExtension: + return BoundingBoxesExtension() diff --git a/ComfyUI/comfy_extras/nodes_camera_trajectory.py b/ComfyUI/comfy_extras/nodes_camera_trajectory.py new file mode 100644 index 0000000000000000000000000000000000000000..280d136af37d9c0443735d0a4af0a509f762a7f7 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_camera_trajectory.py @@ -0,0 +1,239 @@ +import nodes +import torch +import numpy as np +from einops import rearrange +from typing_extensions import override +import comfy.model_management + +from comfy_api.latest import ComfyExtension, io + + +CAMERA_DICT = { + "base_T_norm": 1.5, + "base_angle": np.pi/3, + "Static": { "angle":[0., 0., 0.], "T":[0., 0., 0.]}, + "Pan Up": { "angle":[0., 0., 0.], "T":[0., -1., 0.]}, + "Pan Down": { "angle":[0., 0., 0.], "T":[0.,1.,0.]}, + "Pan Left": { "angle":[0., 0., 0.], "T":[-1.,0.,0.]}, + "Pan Right": { "angle":[0., 0., 0.], "T": [1.,0.,0.]}, + "Zoom In": { "angle":[0., 0., 0.], "T": [0.,0.,2.]}, + "Zoom Out": { "angle":[0., 0., 0.], "T": [0.,0.,-2.]}, + "Anti Clockwise (ACW)": { "angle": [0., 0., -1.], "T":[0., 0., 0.]}, + "ClockWise (CW)": { "angle": [0., 0., 1.], "T":[0., 0., 0.]}, +} + + +def process_pose_params(cam_params, width=672, height=384, original_pose_width=1280, original_pose_height=720, device='cpu'): + + def get_relative_pose(cam_params): + """Copied from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + abs_w2cs = [cam_param.w2c_mat for cam_param in cam_params] + abs_c2ws = [cam_param.c2w_mat for cam_param in cam_params] + cam_to_origin = 0 + target_cam_c2w = np.array([ + [1, 0, 0, 0], + [0, 1, 0, -cam_to_origin], + [0, 0, 1, 0], + [0, 0, 0, 1] + ]) + abs2rel = target_cam_c2w @ abs_w2cs[0] + ret_poses = [target_cam_c2w, ] + [abs2rel @ abs_c2w for abs_c2w in abs_c2ws[1:]] + ret_poses = np.array(ret_poses, dtype=np.float32) + return ret_poses + + """Modified from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + cam_params = [Camera(cam_param) for cam_param in cam_params] + + sample_wh_ratio = width / height + pose_wh_ratio = original_pose_width / original_pose_height # Assuming placeholder ratios, change as needed + + if pose_wh_ratio > sample_wh_ratio: + resized_ori_w = height * pose_wh_ratio + for cam_param in cam_params: + cam_param.fx = resized_ori_w * cam_param.fx / width + else: + resized_ori_h = width / pose_wh_ratio + for cam_param in cam_params: + cam_param.fy = resized_ori_h * cam_param.fy / height + + intrinsic = np.asarray([[cam_param.fx * width, + cam_param.fy * height, + cam_param.cx * width, + cam_param.cy * height] + for cam_param in cam_params], dtype=np.float32) + + K = torch.as_tensor(intrinsic)[None] # [1, 1, 4] + c2ws = get_relative_pose(cam_params) # Assuming this function is defined elsewhere + c2ws = torch.as_tensor(c2ws)[None] # [1, n_frame, 4, 4] + plucker_embedding = ray_condition(K, c2ws, height, width, device=device)[0].permute(0, 3, 1, 2).contiguous() # V, 6, H, W + plucker_embedding = plucker_embedding[None] + plucker_embedding = rearrange(plucker_embedding, "b f c h w -> b f h w c")[0] + return plucker_embedding + +class Camera(object): + """Copied from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + def __init__(self, entry): + fx, fy, cx, cy = entry[1:5] + self.fx = fx + self.fy = fy + self.cx = cx + self.cy = cy + c2w_mat = np.array(entry[7:]).reshape(4, 4) + self.c2w_mat = c2w_mat + self.w2c_mat = np.linalg.inv(c2w_mat) + +def ray_condition(K, c2w, H, W, device): + """Copied from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + # c2w: B, V, 4, 4 + # K: B, V, 4 + + B = K.shape[0] + + j, i = torch.meshgrid( + torch.linspace(0, H - 1, H, device=device, dtype=c2w.dtype), + torch.linspace(0, W - 1, W, device=device, dtype=c2w.dtype), + indexing='ij' + ) + i = i.reshape([1, 1, H * W]).expand([B, 1, H * W]) + 0.5 # [B, HxW] + j = j.reshape([1, 1, H * W]).expand([B, 1, H * W]) + 0.5 # [B, HxW] + + fx, fy, cx, cy = K.chunk(4, dim=-1) # B,V, 1 + + zs = torch.ones_like(i) # [B, HxW] + xs = (i - cx) / fx * zs + ys = (j - cy) / fy * zs + zs = zs.expand_as(ys) + + directions = torch.stack((xs, ys, zs), dim=-1) # B, V, HW, 3 + directions = directions / directions.norm(dim=-1, keepdim=True) # B, V, HW, 3 + + rays_d = directions @ c2w[..., :3, :3].transpose(-1, -2) # B, V, 3, HW + rays_o = c2w[..., :3, 3] # B, V, 3 + rays_o = rays_o[:, :, None].expand_as(rays_d) # B, V, 3, HW + # c2w @ dirctions + rays_dxo = torch.cross(rays_o, rays_d) + plucker = torch.cat([rays_dxo, rays_d], dim=-1) + plucker = plucker.reshape(B, c2w.shape[1], H, W, 6) # B, V, H, W, 6 + # plucker = plucker.permute(0, 1, 4, 2, 3) + return plucker + +def get_camera_motion(angle, T, speed, n=81): + def compute_R_form_rad_angle(angles): + theta_x, theta_y, theta_z = angles + Rx = np.array([[1, 0, 0], + [0, np.cos(theta_x), -np.sin(theta_x)], + [0, np.sin(theta_x), np.cos(theta_x)]]) + + Ry = np.array([[np.cos(theta_y), 0, np.sin(theta_y)], + [0, 1, 0], + [-np.sin(theta_y), 0, np.cos(theta_y)]]) + + Rz = np.array([[np.cos(theta_z), -np.sin(theta_z), 0], + [np.sin(theta_z), np.cos(theta_z), 0], + [0, 0, 1]]) + + R = np.dot(Rz, np.dot(Ry, Rx)) + return R + RT = [] + for i in range(n): + _angle = (i/n)*speed*(CAMERA_DICT["base_angle"])*angle + R = compute_R_form_rad_angle(_angle) + _T=(i/n)*speed*(CAMERA_DICT["base_T_norm"])*(T.reshape(3,1)) + _RT = np.concatenate([R,_T], axis=1) + RT.append(_RT) + RT = np.stack(RT) + return RT + +class WanCameraEmbedding(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="WanCameraEmbedding", + category="model/conditioning/wan/camera", + inputs=[ + io.Combo.Input( + "camera_pose", + options=[ + "Static", + "Pan Up", + "Pan Down", + "Pan Left", + "Pan Right", + "Zoom In", + "Zoom Out", + "Anti Clockwise (ACW)", + "ClockWise (CW)", + ], + default="Static", + ), + io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4), + io.Float.Input("speed", default=1.0, min=0, max=10.0, step=0.1, optional=True), + io.Float.Input("fx", default=0.5, min=0, max=1, step=0.000000001, optional=True, advanced=True), + io.Float.Input("fy", default=0.5, min=0, max=1, step=0.000000001, optional=True, advanced=True), + io.Float.Input("cx", default=0.5, min=0, max=1, step=0.01, optional=True, advanced=True), + io.Float.Input("cy", default=0.5, min=0, max=1, step=0.01, optional=True, advanced=True), + ], + outputs=[ + io.WanCameraEmbedding.Output(display_name="camera_embedding"), + io.Int.Output(display_name="width"), + io.Int.Output(display_name="height"), + io.Int.Output(display_name="length"), + ], + ) + + @classmethod + def execute(cls, camera_pose, width, height, length, speed=1.0, fx=0.5, fy=0.5, cx=0.5, cy=0.5) -> io.NodeOutput: + """ + Use Camera trajectory as extrinsic parameters to calculate Plücker embeddings (Sitzmannet al., 2021) + Adapted from https://github.com/aigc-apps/VideoX-Fun/blob/main/comfyui/comfyui_nodes.py + """ + motion_list = [camera_pose] + speed = speed + angle = np.array(CAMERA_DICT[motion_list[0]]["angle"]) + T = np.array(CAMERA_DICT[motion_list[0]]["T"]) + RT = get_camera_motion(angle, T, speed, length) + + trajs=[] + for cp in RT.tolist(): + traj=[fx,fy,cx,cy,0,0] + traj.extend(cp[0]) + traj.extend(cp[1]) + traj.extend(cp[2]) + traj.extend([0,0,0,1]) + trajs.append(traj) + + cam_params = np.array([[float(x) for x in pose] for pose in trajs]) + cam_params = np.concatenate([np.zeros_like(cam_params[:, :1]), cam_params], 1) + control_camera_video = process_pose_params(cam_params, width=width, height=height) + control_camera_video = control_camera_video.permute([3, 0, 1, 2]).unsqueeze(0).to(device=comfy.model_management.intermediate_device()) + + control_camera_video = torch.concat( + [ + torch.repeat_interleave(control_camera_video[:, :, 0:1], repeats=4, dim=2), + control_camera_video[:, :, 1:] + ], dim=2 + ).transpose(1, 2) + + # Reshape, transpose, and view into desired shape + b, f, c, h, w = control_camera_video.shape + control_camera_video = control_camera_video.contiguous().view(b, f // 4, 4, c, h, w).transpose(2, 3) + control_camera_video = control_camera_video.contiguous().view(b, f // 4, c * 4, h, w).transpose(1, 2) + + return io.NodeOutput(control_camera_video, width, height, length) + + +class CameraTrajectoryExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + WanCameraEmbedding, + ] + +async def comfy_entrypoint() -> CameraTrajectoryExtension: + return CameraTrajectoryExtension() diff --git a/ComfyUI/comfy_extras/nodes_canny.py b/ComfyUI/comfy_extras/nodes_canny.py new file mode 100644 index 0000000000000000000000000000000000000000..462f6fea0d571e904e3b9e5ff442bdd3b800777e --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_canny.py @@ -0,0 +1,45 @@ +from kornia.filters import canny +from typing_extensions import override + +import comfy.model_management +from comfy_api.latest import ComfyExtension, io +import torch + + +class Canny(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="Canny", + display_name="Detect Edges (Canny)", + search_aliases=["edge detection", "outline", "contour detection", "line art"], + category="image/filters", + essentials_category="Image Tools", + inputs=[ + io.Image.Input("image"), + io.Float.Input("low_threshold", default=0.4, min=0.01, max=0.99, step=0.01), + io.Float.Input("high_threshold", default=0.8, min=0.01, max=0.99, step=0.01), + ], + outputs=[io.Image.Output()], + ) + + @classmethod + def detect_edge(cls, image, low_threshold, high_threshold): + # Deprecated: use the V3 schema's `execute` method instead of this. + return cls.execute(image, low_threshold, high_threshold) + + @classmethod + def execute(cls, image, low_threshold, high_threshold) -> io.NodeOutput: + output = canny(image.to(device=comfy.model_management.get_torch_device(), dtype=torch.float32).movedim(-1, 1), low_threshold, high_threshold) + img_out = output[1].to(device=comfy.model_management.intermediate_device(), dtype=comfy.model_management.intermediate_dtype()).repeat(1, 3, 1, 1).movedim(1, -1) + return io.NodeOutput(img_out) + + +class CannyExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [Canny] + + +async def comfy_entrypoint() -> CannyExtension: + return CannyExtension() diff --git a/ComfyUI/comfy_extras/nodes_cfg.py b/ComfyUI/comfy_extras/nodes_cfg.py new file mode 100644 index 0000000000000000000000000000000000000000..b585c560f6a77ba4bb78f1e842a24a361a7f319e --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_cfg.py @@ -0,0 +1,122 @@ +from typing_extensions import override + +import torch + +from comfy_api.latest import ComfyExtension, io + + +# https://github.com/WeichenFan/CFG-Zero-star +def optimized_scale(positive, negative): + positive_flat = positive.reshape(positive.shape[0], -1) + negative_flat = negative.reshape(negative.shape[0], -1) + + # Calculate dot production + dot_product = torch.sum(positive_flat * negative_flat, dim=1, keepdim=True) + + # Squared norm of uncondition + squared_norm = torch.sum(negative_flat ** 2, dim=1, keepdim=True) + 1e-8 + + # st_star = v_cond^T * v_uncond / ||v_uncond||^2 + st_star = dot_product / squared_norm + + return st_star.reshape([positive.shape[0]] + [1] * (positive.ndim - 1)) + +class CFGZeroStar(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="CFGZeroStar", + category="advanced/guidance", + inputs=[ + io.Model.Input("model"), + ], + outputs=[io.Model.Output(display_name="patched_model")], + ) + + @classmethod + def execute(cls, model) -> io.NodeOutput: + m = model.clone() + def cfg_zero_star(args): + guidance_scale = args['cond_scale'] + x = args['input'] + cond_p = args['cond_denoised'] + uncond_p = args['uncond_denoised'] + out = args["denoised"] + alpha = optimized_scale(x - cond_p, x - uncond_p) + + return out + uncond_p * (alpha - 1.0) + guidance_scale * uncond_p * (1.0 - alpha) + m.set_model_sampler_post_cfg_function(cfg_zero_star) + return io.NodeOutput(m) + +class CFGNorm(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="CFGNorm", + category="advanced/guidance", + inputs=[ + io.Model.Input("model"), + io.Float.Input("strength", default=1.0, min=0.0, max=100.0, step=0.01), + io.Boolean.Input( + "pre_cfg", + default=False, + optional=True, + tooltip=( + "If true, rescale the combined noise BEFORE the sampler's CFG combine, " + "without clamping (can amplify). Matches the norm-scaled CFG used by " + "models like Lens. Default false keeps the original post-CFG x0-space " + "attenuate-only behavior." + ), + ), + ], + outputs=[io.Model.Output(display_name="patched_model")], + is_experimental=True, + ) + + @classmethod + def execute(cls, model, strength, pre_cfg=False) -> io.NodeOutput: + m = model.clone() + if pre_cfg: + def cfg_norm_pre(args): + cond = args["cond"] + uncond = args["uncond"] + cond_scale = args["cond_scale"] + comb = uncond + cond_scale * (cond - uncond) + cond_norm = torch.linalg.vector_norm(cond, dim=1, keepdim=True) + comb_norm = torch.linalg.vector_norm(comb, dim=1, keepdim=True) + rescale = torch.where( + comb_norm > 0, + cond_norm / comb_norm.clamp_min(1e-12), + torch.ones_like(comb_norm), + ) + rescaled = comb * rescale + # strength blends back toward standard linear CFG (1.0 = full rescale). + if strength != 1.0: + rescaled = strength * rescaled + (1.0 - strength) * comb + return rescaled + m.set_model_sampler_cfg_function(cfg_norm_pre) + else: + def cfg_norm(args): + cond_p = args['cond_denoised'] + pred_text_ = args["denoised"] + + norm_full_cond = torch.norm(cond_p, dim=1, keepdim=True) + norm_pred_text = torch.norm(pred_text_, dim=1, keepdim=True) + scale = (norm_full_cond / (norm_pred_text + 1e-8)).clamp(min=0.0, max=1.0) + return pred_text_ * scale * strength + + m.set_model_sampler_post_cfg_function(cfg_norm) + return io.NodeOutput(m) + + +class CfgExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + CFGZeroStar, + CFGNorm, + ] + + +async def comfy_entrypoint() -> CfgExtension: + return CfgExtension() diff --git a/ComfyUI/comfy_extras/nodes_chroma_radiance.py b/ComfyUI/comfy_extras/nodes_chroma_radiance.py new file mode 100644 index 0000000000000000000000000000000000000000..059344f3cd2272ef5b58791e6880135ff905edbd --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_chroma_radiance.py @@ -0,0 +1,127 @@ +from typing_extensions import override +from typing import Callable + +import torch + +import comfy.model_management +from comfy_api.latest import ComfyExtension, io + +import nodes + +class EmptyChromaRadianceLatentImage(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="EmptyChromaRadianceLatentImage", + category="model/latent/chroma radiance", + inputs=[ + io.Int.Input(id="width", default=1024, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input(id="height", default=1024, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input(id="batch_size", default=1, min=1, max=4096), + ], + outputs=[io.Latent().Output()], + ) + + @classmethod + def execute(cls, *, width: int, height: int, batch_size: int=1) -> io.NodeOutput: + latent = torch.zeros((batch_size, 3, height, width), device=comfy.model_management.intermediate_device()) + return io.NodeOutput({"samples":latent}) + + +class ChromaRadianceOptions(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="ChromaRadianceOptions", + category="model/patch/chroma radiance", + description="Allows setting advanced options for the Chroma Radiance model.", + inputs=[ + io.Model.Input(id="model"), + io.Boolean.Input( + id="preserve_wrapper", + default=True, + tooltip="When enabled, will delegate to an existing model function wrapper if it exists. Generally should be left enabled.", + ), + io.Float.Input( + id="start_sigma", + default=1.0, + min=0.0, + max=1.0, + tooltip="First sigma that these options will be in effect.", + advanced=True, + ), + io.Float.Input( + id="end_sigma", + default=0.0, + min=0.0, + max=1.0, + tooltip="Last sigma that these options will be in effect.", + advanced=True, + ), + io.Int.Input( + id="nerf_tile_size", + default=-1, + min=-1, + tooltip="Allows overriding the default NeRF tile size. -1 means use the default (32). 0 means use non-tiling mode (may require a lot of VRAM).", + advanced=True, + ), + io.Boolean.Input( + id="force_sequential_txt_ids", + default=False, + tooltip="Force usage of sequential text token IDs instead of zeroes. Should be used for checkpoints from 2026-05-22 to 2026-06-01 that are trained in this way but do not contain the __sequential__ key in the state dict.", + advanced=True, + ), + ], + outputs=[io.Model.Output()], + ) + + @classmethod + def execute( + cls, + *, + model: io.Model.Type, + preserve_wrapper: bool, + start_sigma: float, + end_sigma: float, + nerf_tile_size: int, + force_sequential_txt_ids: bool, + ) -> io.NodeOutput: + radiance_options = {} + if nerf_tile_size >= 0: + radiance_options["nerf_tile_size"] = nerf_tile_size + + if force_sequential_txt_ids: + radiance_options["use_sequential_txt_ids"] = True + + if not radiance_options: + return io.NodeOutput(model) + + old_wrapper = model.model_options.get("model_function_wrapper") + + def model_function_wrapper(apply_model: Callable, args: dict) -> torch.Tensor: + c = args["c"].copy() + sigma = args["timestep"].max().detach().cpu().item() + if end_sigma <= sigma <= start_sigma: + transformer_options = c.get("transformer_options", {}).copy() + transformer_options["chroma_radiance_options"] = radiance_options.copy() + c["transformer_options"] = transformer_options + if not (preserve_wrapper and old_wrapper): + return apply_model(args["input"], args["timestep"], **c) + return old_wrapper(apply_model, args | {"c": c}) + + model = model.clone() + model.set_model_unet_function_wrapper(model_function_wrapper) + return io.NodeOutput(model) + + +class ChromaRadianceExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + EmptyChromaRadianceLatentImage, + ChromaRadianceOptions, + ] + + +async def comfy_entrypoint() -> ChromaRadianceExtension: + return ChromaRadianceExtension() diff --git a/ComfyUI/comfy_extras/nodes_clip_sdxl.py b/ComfyUI/comfy_extras/nodes_clip_sdxl.py new file mode 100644 index 0000000000000000000000000000000000000000..08fbbd827aa46d7c00b70a49acca92973b8155a4 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_clip_sdxl.py @@ -0,0 +1,73 @@ +from typing_extensions import override + +import nodes +from comfy_api.latest import ComfyExtension, io + + +class CLIPTextEncodeSDXLRefiner(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="CLIPTextEncodeSDXLRefiner", + display_name="CLIP Text Encode (SDXL Refiner)", + category="model/conditioning/stable diffusion", + inputs=[ + io.Float.Input("ascore", default=6.0, min=0.0, max=1000.0, step=0.01), + io.Int.Input("width", default=1024, min=0, max=nodes.MAX_RESOLUTION), + io.Int.Input("height", default=1024, min=0, max=nodes.MAX_RESOLUTION), + io.String.Input("text", multiline=True, dynamic_prompts=True), + io.Clip.Input("clip"), + ], + outputs=[io.Conditioning.Output()], + ) + + @classmethod + def execute(cls, clip, ascore, width, height, text) -> io.NodeOutput: + tokens = clip.tokenize(text) + return io.NodeOutput(clip.encode_from_tokens_scheduled(tokens, add_dict={"aesthetic_score": ascore, "width": width, "height": height})) + +class CLIPTextEncodeSDXL(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="CLIPTextEncodeSDXL", + display_name="CLIP Text Encode (SDXL)", + category="model/conditioning/stable diffusion", + inputs=[ + io.Clip.Input("clip"), + io.Int.Input("width", default=1024, min=0, max=nodes.MAX_RESOLUTION), + io.Int.Input("height", default=1024, min=0, max=nodes.MAX_RESOLUTION), + io.Int.Input("crop_w", default=0, min=0, max=nodes.MAX_RESOLUTION, advanced=True), + io.Int.Input("crop_h", default=0, min=0, max=nodes.MAX_RESOLUTION, advanced=True), + io.Int.Input("target_width", default=1024, min=0, max=nodes.MAX_RESOLUTION), + io.Int.Input("target_height", default=1024, min=0, max=nodes.MAX_RESOLUTION), + io.String.Input("text_g", multiline=True, dynamic_prompts=True), + io.String.Input("text_l", multiline=True, dynamic_prompts=True), + ], + outputs=[io.Conditioning.Output()], + ) + + @classmethod + def execute(cls, clip, width, height, crop_w, crop_h, target_width, target_height, text_g, text_l) -> io.NodeOutput: + tokens = clip.tokenize(text_g) + tokens["l"] = clip.tokenize(text_l)["l"] + if len(tokens["l"]) != len(tokens["g"]): + empty = clip.tokenize("") + while len(tokens["l"]) < len(tokens["g"]): + tokens["l"] += empty["l"] + while len(tokens["l"]) > len(tokens["g"]): + tokens["g"] += empty["g"] + return io.NodeOutput(clip.encode_from_tokens_scheduled(tokens, add_dict={"width": width, "height": height, "crop_w": crop_w, "crop_h": crop_h, "target_width": target_width, "target_height": target_height})) + + +class ClipSdxlExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + CLIPTextEncodeSDXLRefiner, + CLIPTextEncodeSDXL, + ] + + +async def comfy_entrypoint() -> ClipSdxlExtension: + return ClipSdxlExtension() diff --git a/ComfyUI/comfy_extras/nodes_color.py b/ComfyUI/comfy_extras/nodes_color.py new file mode 100644 index 0000000000000000000000000000000000000000..6d10b26f4a04dad7730c185fe56c487c1b3d4829 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_color.py @@ -0,0 +1,52 @@ +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io +from comfy_extras.color_util import hex_to_rgb + + +class ColorToRGBInt(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="ColorToRGBInt", + display_name="Color Picker", + category="utilities", + description="Return a color RGB integer value and hexadecimal representation.", + inputs=[ + io.Color.Input("color"), + ], + outputs=[ + io.Int.Output(display_name="rgb_int"), + io.Color.Output(display_name="hex"), + io.Float.Output(display_name="alpha"), + ], + ) + + @classmethod + def execute(cls, color: str) -> io.NodeOutput: + # expect format #RRGGBB or #RRGGBBAA + if len(color) not in (7, 9) or color[0] != "#": + raise ValueError("Color must be in format #RRGGBB or #RRGGBBAA") + try: + int(color[1:], 16) + except ValueError: + raise ValueError("Color must be in format #RRGGBB or #RRGGBBAA") from None + + alpha = 1.0 + if len(color) == 9: + alpha = int(color[7:9], 16) / 255.0 + color = color[:7] + + r, g, b = hex_to_rgb(color) + + rgb_int = r * 256 * 256 + g * 256 + b + return io.NodeOutput(rgb_int, color, alpha) + + +class ColorExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ColorToRGBInt] + + +async def comfy_entrypoint() -> ColorExtension: + return ColorExtension() diff --git a/ComfyUI/comfy_extras/nodes_compositing.py b/ComfyUI/comfy_extras/nodes_compositing.py new file mode 100644 index 0000000000000000000000000000000000000000..8fcbe720e7398d24d7dbc957787f68ec0d123713 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_compositing.py @@ -0,0 +1,223 @@ +import torch +import comfy.utils +from enum import Enum +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io + + +def resize_mask(mask, shape): + return torch.nn.functional.interpolate(mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])), size=(shape[0], shape[1]), mode="bilinear").squeeze(1) + +class PorterDuffMode(Enum): + ADD = 0 + CLEAR = 1 + DARKEN = 2 + DST = 3 + DST_ATOP = 4 + DST_IN = 5 + DST_OUT = 6 + DST_OVER = 7 + LIGHTEN = 8 + MULTIPLY = 9 + OVERLAY = 10 + SCREEN = 11 + SRC = 12 + SRC_ATOP = 13 + SRC_IN = 14 + SRC_OUT = 15 + SRC_OVER = 16 + XOR = 17 + + +def porter_duff_composite(src_image: torch.Tensor, src_alpha: torch.Tensor, dst_image: torch.Tensor, dst_alpha: torch.Tensor, mode: PorterDuffMode): + # convert mask to alpha + src_alpha = 1 - src_alpha + dst_alpha = 1 - dst_alpha + # premultiply alpha + src_image = src_image * src_alpha + dst_image = dst_image * dst_alpha + + # composite ops below assume alpha-premultiplied images + if mode == PorterDuffMode.ADD: + out_alpha = torch.clamp(src_alpha + dst_alpha, 0, 1) + out_image = torch.clamp(src_image + dst_image, 0, 1) + elif mode == PorterDuffMode.CLEAR: + out_alpha = torch.zeros_like(dst_alpha) + out_image = torch.zeros_like(dst_image) + elif mode == PorterDuffMode.DARKEN: + out_alpha = src_alpha + dst_alpha - src_alpha * dst_alpha + out_image = (1 - dst_alpha) * src_image + (1 - src_alpha) * dst_image + torch.min(src_image, dst_image) + elif mode == PorterDuffMode.DST: + out_alpha = dst_alpha + out_image = dst_image + elif mode == PorterDuffMode.DST_ATOP: + out_alpha = src_alpha + out_image = src_alpha * dst_image + (1 - dst_alpha) * src_image + elif mode == PorterDuffMode.DST_IN: + out_alpha = src_alpha * dst_alpha + out_image = dst_image * src_alpha + elif mode == PorterDuffMode.DST_OUT: + out_alpha = (1 - src_alpha) * dst_alpha + out_image = (1 - src_alpha) * dst_image + elif mode == PorterDuffMode.DST_OVER: + out_alpha = dst_alpha + (1 - dst_alpha) * src_alpha + out_image = dst_image + (1 - dst_alpha) * src_image + elif mode == PorterDuffMode.LIGHTEN: + out_alpha = src_alpha + dst_alpha - src_alpha * dst_alpha + out_image = (1 - dst_alpha) * src_image + (1 - src_alpha) * dst_image + torch.max(src_image, dst_image) + elif mode == PorterDuffMode.MULTIPLY: + out_alpha = src_alpha * dst_alpha + out_image = src_image * dst_image + elif mode == PorterDuffMode.OVERLAY: + out_alpha = src_alpha + dst_alpha - src_alpha * dst_alpha + out_image = torch.where(2 * dst_image < dst_alpha, 2 * src_image * dst_image, + src_alpha * dst_alpha - 2 * (dst_alpha - src_image) * (src_alpha - dst_image)) + elif mode == PorterDuffMode.SCREEN: + out_alpha = src_alpha + dst_alpha - src_alpha * dst_alpha + out_image = src_image + dst_image - src_image * dst_image + elif mode == PorterDuffMode.SRC: + out_alpha = src_alpha + out_image = src_image + elif mode == PorterDuffMode.SRC_ATOP: + out_alpha = dst_alpha + out_image = dst_alpha * src_image + (1 - src_alpha) * dst_image + elif mode == PorterDuffMode.SRC_IN: + out_alpha = src_alpha * dst_alpha + out_image = src_image * dst_alpha + elif mode == PorterDuffMode.SRC_OUT: + out_alpha = (1 - dst_alpha) * src_alpha + out_image = (1 - dst_alpha) * src_image + elif mode == PorterDuffMode.SRC_OVER: + out_alpha = src_alpha + (1 - src_alpha) * dst_alpha + out_image = src_image + (1 - src_alpha) * dst_image + elif mode == PorterDuffMode.XOR: + out_alpha = (1 - dst_alpha) * src_alpha + (1 - src_alpha) * dst_alpha + out_image = (1 - dst_alpha) * src_image + (1 - src_alpha) * dst_image + else: + return None, None + + # back to non-premultiplied alpha + out_image = torch.where(out_alpha > 1e-5, out_image / out_alpha, torch.zeros_like(out_image)) + out_image = torch.clamp(out_image, 0, 1) + # convert alpha to mask + out_alpha = 1 - out_alpha + return out_image, out_alpha + + +class PorterDuffImageComposite(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="PorterDuffImageComposite", + search_aliases=["alpha composite", "blend modes", "layer blend", "transparency blend"], + display_name="Porter-Duff Image Composite", + category="image/compositing", + inputs=[ + io.Image.Input("source"), + io.Mask.Input("source_alpha"), + io.Image.Input("destination"), + io.Mask.Input("destination_alpha"), + io.Combo.Input("mode", options=[mode.name for mode in PorterDuffMode], default=PorterDuffMode.DST.name), + ], + outputs=[ + io.Image.Output(), + io.Mask.Output(), + ], + ) + + @classmethod + def execute(cls, source: torch.Tensor, source_alpha: torch.Tensor, destination: torch.Tensor, destination_alpha: torch.Tensor, mode) -> io.NodeOutput: + batch_size = min(len(source), len(source_alpha), len(destination), len(destination_alpha)) + out_images = [] + out_alphas = [] + + for i in range(batch_size): + src_image = source[i] + dst_image = destination[i] + + assert src_image.shape[2] == dst_image.shape[2] # inputs need to have same number of channels + + src_alpha = source_alpha[i].unsqueeze(2) + dst_alpha = destination_alpha[i].unsqueeze(2) + + if dst_alpha.shape[:2] != dst_image.shape[:2]: + upscale_input = dst_alpha.unsqueeze(0).permute(0, 3, 1, 2) + upscale_output = comfy.utils.common_upscale(upscale_input, dst_image.shape[1], dst_image.shape[0], upscale_method='bicubic', crop='center') + dst_alpha = upscale_output.permute(0, 2, 3, 1).squeeze(0) + if src_image.shape != dst_image.shape: + upscale_input = src_image.unsqueeze(0).permute(0, 3, 1, 2) + upscale_output = comfy.utils.common_upscale(upscale_input, dst_image.shape[1], dst_image.shape[0], upscale_method='bicubic', crop='center') + src_image = upscale_output.permute(0, 2, 3, 1).squeeze(0) + if src_alpha.shape != dst_alpha.shape: + upscale_input = src_alpha.unsqueeze(0).permute(0, 3, 1, 2) + upscale_output = comfy.utils.common_upscale(upscale_input, dst_alpha.shape[1], dst_alpha.shape[0], upscale_method='bicubic', crop='center') + src_alpha = upscale_output.permute(0, 2, 3, 1).squeeze(0) + + out_image, out_alpha = porter_duff_composite(src_image, src_alpha, dst_image, dst_alpha, PorterDuffMode[mode]) + + out_images.append(out_image) + out_alphas.append(out_alpha.squeeze(2)) + + return io.NodeOutput(torch.stack(out_images), torch.stack(out_alphas)) + + +class SplitImageWithAlpha(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SplitImageWithAlpha", + search_aliases=["extract alpha", "separate transparency", "remove alpha"], + display_name="Split Image with Alpha", + category="image/compositing", + inputs=[ + io.Image.Input("image"), + ], + outputs=[ + io.Image.Output(), + io.Mask.Output(), + ], + ) + + @classmethod + def execute(cls, image: torch.Tensor) -> io.NodeOutput: + out_images = [i[:,:,:3] for i in image] + out_alphas = [i[:,:,3] if i.shape[2] > 3 else torch.ones_like(i[:,:,0]) for i in image] + return io.NodeOutput(torch.stack(out_images), 1.0 - torch.stack(out_alphas)) + + +class JoinImageWithAlpha(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="JoinImageWithAlpha", + search_aliases=["add transparency", "apply alpha", "composite alpha", "RGBA"], + display_name="Join Image with Alpha", + category="image/compositing", + inputs=[ + io.Image.Input("image"), + io.Mask.Input("alpha"), + ], + outputs=[io.Image.Output()], + ) + + @classmethod + def execute(cls, image: torch.Tensor, alpha: torch.Tensor) -> io.NodeOutput: + batch_size = max(len(image), len(alpha)) + alpha = 1.0 - resize_mask(alpha.to(image), image.shape[1:]) + alpha = comfy.utils.repeat_to_batch_size(alpha, batch_size) + image = comfy.utils.repeat_to_batch_size(image, batch_size) + return io.NodeOutput(torch.cat((image[..., :3], alpha.unsqueeze(-1)), dim=-1)) + + +class CompositingExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + PorterDuffImageComposite, + SplitImageWithAlpha, + JoinImageWithAlpha, + ] + + +async def comfy_entrypoint() -> CompositingExtension: + return CompositingExtension() diff --git a/ComfyUI/comfy_extras/nodes_cond.py b/ComfyUI/comfy_extras/nodes_cond.py new file mode 100644 index 0000000000000000000000000000000000000000..c8091b7a4d7d620d1435c5eaef77fc00e2c7925f --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_cond.py @@ -0,0 +1,70 @@ +from typing_extensions import override + +from comfy_api.latest import ComfyExtension, io + + +class CLIPTextEncodeControlnet(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="CLIPTextEncodeControlnet", + display_name="CLIP Text Encode (Controlnet)", + category="model/conditioning", + inputs=[ + io.Clip.Input("clip"), + io.Conditioning.Input("conditioning"), + io.String.Input("text", multiline=True, dynamic_prompts=True), + ], + outputs=[io.Conditioning.Output()], + is_experimental=True, + ) + + @classmethod + def execute(cls, clip, conditioning, text) -> io.NodeOutput: + tokens = clip.tokenize(text) + cond, pooled = clip.encode_from_tokens(tokens, return_pooled=True) + c = [] + for t in conditioning: + n = [t[0], t[1].copy()] + n[1]['cross_attn_controlnet'] = cond + n[1]['pooled_output_controlnet'] = pooled + c.append(n) + return io.NodeOutput(c) + +class T5TokenizerOptions(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="T5TokenizerOptions", + display_name="T5 Tokenizer Options", + category="model/conditioning", + inputs=[ + io.Clip.Input("clip"), + io.Int.Input("min_padding", default=0, min=0, max=10000, step=1), + io.Int.Input("min_length", default=0, min=0, max=10000, step=1), + ], + outputs=[io.Clip.Output()], + is_experimental=True, + ) + + @classmethod + def execute(cls, clip, min_padding, min_length) -> io.NodeOutput: + clip = clip.clone() + for t5_type in ["t5xxl", "pile_t5xl", "t5base", "mt5xl", "umt5xxl"]: + clip.set_tokenizer_option("{}_min_padding".format(t5_type), min_padding) + clip.set_tokenizer_option("{}_min_length".format(t5_type), min_length) + + return io.NodeOutput(clip) + + +class CondExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + CLIPTextEncodeControlnet, + T5TokenizerOptions, + ] + + +async def comfy_entrypoint() -> CondExtension: + return CondExtension() diff --git a/ComfyUI/comfy_extras/nodes_context_windows.py b/ComfyUI/comfy_extras/nodes_context_windows.py new file mode 100644 index 0000000000000000000000000000000000000000..15d2dc506e075eda7a90e5364c1e62d32c5de388 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_context_windows.py @@ -0,0 +1,146 @@ +from comfy_api.latest import ComfyExtension, io +import comfy.context_windows +import nodes + + +class ContextWindowsManualNode(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="ContextWindowsManual", + display_name="Context Windows (Manual)", + category="model/patch", + description="Manually set context windows.", + inputs=[ + io.Model.Input("model", tooltip="The model to apply context windows to during sampling."), + io.Int.Input("context_length", min=1, default=16, tooltip="The length of the context window."), + io.Int.Input("context_overlap", min=0, default=4, tooltip="The overlap of the context window."), + io.Combo.Input("context_schedule", options=[ + comfy.context_windows.ContextSchedules.STATIC_STANDARD, + comfy.context_windows.ContextSchedules.UNIFORM_STANDARD, + comfy.context_windows.ContextSchedules.UNIFORM_LOOPED, + comfy.context_windows.ContextSchedules.BATCHED, + ], default=comfy.context_windows.ContextSchedules.STATIC_STANDARD, tooltip="Step-dependent scheduling algorithm for context windows."), + io.Int.Input("context_stride", min=1, default=1, tooltip="The stride of the context window; only applicable to uniform schedules."), + io.Boolean.Input("closed_loop", default=False, tooltip="Whether to close the context window loop; only applicable to looped schedules."), + io.Combo.Input("fuse_method", options=comfy.context_windows.ContextFuseMethods.LIST_STATIC, default=comfy.context_windows.ContextFuseMethods.PYRAMID, tooltip="The method to use to fuse the context windows."), + io.Int.Input("dim", min=0, max=5, default=0, tooltip="The dimension to apply the context windows to."), + io.Boolean.Input("freenoise", default=False, tooltip="Whether to apply FreeNoise noise shuffling, improves window blending."), + io.String.Input("cond_retain_index_list", default="", tooltip="List of latent indices to retain in the conditioning tensors for each window. For concat-style I2V models (e.g. Wan I2V, HunyuanVideo I2V, Cosmos I2V, SVD) the encoded start image lives in the c_concat conditioning channels; setting this to '0' will retain that start image content at sub-pos 0 of every window."), + io.Boolean.Input("split_conds_to_windows", default=False, tooltip="Whether to split multiple conditionings (created by ConditionCombine) to each window based on region index."), + io.String.Input("latent_retain_index_list", default="", tooltip="List of latent indices to retain in the noise latent itself for each window. Use for workflows where reference content (e.g. a start image) lives directly in the noise latent rather than in separate conditioning channels (e.g. inplace-style I2V like LTXV, AnimateDiff). Independent of cond_retain_index_list."), + io.Boolean.Input("causal_window_fix", default=True, tooltip="Whether to add a causal fix frame to non-0-indexed context windows."), + ], + outputs=[ + io.Model.Output(tooltip="The model with context windows applied during sampling."), + ], + is_experimental=True, + ) + + @classmethod + def execute(cls, model: io.Model.Type, context_length: int, context_overlap: int, context_schedule: str, context_stride: int, closed_loop: bool, fuse_method: str, dim: int, freenoise: bool, + cond_retain_index_list: list[int]=[], split_conds_to_windows: bool=False, latent_retain_index_list: list[int]=[], causal_window_fix: bool=True) -> io.Model: + model = model.clone() + model.model_options["context_handler"] = comfy.context_windows.IndexListContextHandler( + context_schedule=comfy.context_windows.get_matching_context_schedule(context_schedule), + fuse_method=comfy.context_windows.get_matching_fuse_method(fuse_method), + context_length=context_length, + context_overlap=context_overlap, + context_stride=context_stride, + closed_loop=closed_loop, + dim=dim, + freenoise=freenoise, + cond_retain_index_list=cond_retain_index_list, + split_conds_to_windows=split_conds_to_windows, + latent_retain_index_list=latent_retain_index_list, + causal_window_fix=causal_window_fix, + ) + # make memory usage calculation only take into account the context window latents + comfy.context_windows.create_prepare_sampling_wrapper(model) + if freenoise: # no other use for this wrapper at this time + comfy.context_windows.create_sampler_sample_wrapper(model) + return io.NodeOutput(model) + +class WanContextWindowsManualNode(ContextWindowsManualNode): + @classmethod + def define_schema(cls) -> io.Schema: + schema = super().define_schema() + schema.node_id = "WanContextWindowsManual" + schema.display_name = "WAN Context Windows (Manual)" + schema.display_name = "Wan Context Windows" + schema.description = "Set context windows for Wan-like models." + schema.category="model/patch/wan" + schema.inputs = [ + io.Model.Input("model", tooltip="The model to apply context windows to during sampling."), + io.Int.Input("context_length", min=1, max=nodes.MAX_RESOLUTION, step=4, default=81, tooltip="The length of the context window in real frames. Must be 4*n + 1."), + io.Int.Input("context_overlap", min=0, default=30, tooltip="The overlap of the context window in real frames."), + io.Combo.Input("context_schedule", options=[ + comfy.context_windows.ContextSchedules.STATIC_STANDARD, + comfy.context_windows.ContextSchedules.UNIFORM_STANDARD, + comfy.context_windows.ContextSchedules.UNIFORM_LOOPED, + comfy.context_windows.ContextSchedules.BATCHED, + ], default=comfy.context_windows.ContextSchedules.UNIFORM_STANDARD, tooltip="Step-dependent scheduling algorithm for context windows."), + io.Int.Input("context_stride", min=1, default=1, tooltip="The stride of the context window; only applicable to uniform schedules.", advanced=True), + io.Boolean.Input("closed_loop", default=False, tooltip="Whether to close the context window loop; only applicable to looped schedules.", advanced=True), + io.Combo.Input("fuse_method", options=comfy.context_windows.ContextFuseMethods.LIST_STATIC, default=comfy.context_windows.ContextFuseMethods.PYRAMID, tooltip="The method to use to fuse the context windows."), + io.Boolean.Input("freenoise", default=True, tooltip="Whether to apply FreeNoise noise shuffling, improves window blending.", advanced=True), + io.Boolean.Input("retain_first_frame", default=False, tooltip="Retain the first I2V frame in every context window (may help retain initial reference)."), + io.Boolean.Input("split_conds_to_windows", default=False, tooltip="Whether to split multiple conditionings (created by ConditionCombine) to each window based on region index.", advanced=True), + ] + return schema + + @classmethod + def execute(cls, model: io.Model.Type, context_length: int, context_overlap: int, context_schedule: str, context_stride: int, closed_loop: bool, fuse_method: str, freenoise: bool, + retain_first_frame: bool=False, split_conds_to_windows: bool=False) -> io.Model: + context_length = max(((context_length - 1) // 4) + 1, 1) # at least length 1 + context_overlap = max(context_overlap // 4, 0) # at least overlap 0 + retain_index_list = "0" if retain_first_frame else "" + return super().execute(model, context_length, context_overlap, context_schedule, context_stride, closed_loop, fuse_method, dim=2, freenoise=freenoise, cond_retain_index_list=retain_index_list, split_conds_to_windows=split_conds_to_windows) + + +class LTXVContextWindowsNode(ContextWindowsManualNode): + @classmethod + def define_schema(cls) -> io.Schema: + schema = super().define_schema() + schema.node_id = "LTXVContextWindows" + schema.display_name = "LTXV Context Windows" + schema.description = "Set context windows for LTXV-like models." + schema.inputs = [ + io.Model.Input("model", tooltip="The model to apply context windows to during sampling."), + io.Int.Input("context_length", min=1, max=nodes.MAX_RESOLUTION, step=8, default=145, tooltip="The length of the context window in real frames. Must be 8*n + 1."), + io.Int.Input("context_overlap", min=0, step=8, default=40, tooltip="The overlap of the context window in real frames."), + io.Combo.Input("context_schedule", options=[ + comfy.context_windows.ContextSchedules.STATIC_STANDARD, + comfy.context_windows.ContextSchedules.UNIFORM_STANDARD, + comfy.context_windows.ContextSchedules.UNIFORM_LOOPED, + comfy.context_windows.ContextSchedules.BATCHED, + ], default=comfy.context_windows.ContextSchedules.UNIFORM_STANDARD, tooltip="Step-dependent scheduling algorithm for context windows."), + io.Int.Input("context_stride", min=1, default=1, tooltip="The stride of the context window; only applicable to uniform schedules.", advanced=True), + io.Boolean.Input("closed_loop", default=False, tooltip="Whether to close the context window loop; only applicable to looped schedules.", advanced=True), + io.Combo.Input("fuse_method", options=comfy.context_windows.ContextFuseMethods.LIST_STATIC, default=comfy.context_windows.ContextFuseMethods.PYRAMID, tooltip="The method to use to fuse the context windows."), + io.Boolean.Input("freenoise", default=True, tooltip="Whether to apply FreeNoise noise shuffling, improves window blending.", advanced=True), + io.Boolean.Input("retain_first_frame", default=False, tooltip="Retain the first latent frame in every context window (may help retain initial reference)."), + io.Boolean.Input("split_conds_to_windows", default=False, tooltip="Whether to split multiple conditionings (created by ConditionCombine) to each window based on region index.", advanced=True), + ] + return schema + + @classmethod + def execute(cls, model: io.Model.Type, context_length: int, context_overlap: int, context_schedule: str, fuse_method: str, freenoise: bool, + retain_first_frame: bool=False, split_conds_to_windows: bool=False, context_stride: int=1, closed_loop: bool=False) -> io.Model: + context_length = max(((context_length - 1) // 8) + 1, 1) # at least length 1 + context_overlap = max(context_overlap // 8, 0) # at least overlap 0 + retain_index_list = "0" if retain_first_frame else "" + return super().execute(model, context_length, context_overlap, context_schedule, context_stride, closed_loop, fuse_method, dim=2, freenoise=freenoise, + cond_retain_index_list=retain_index_list, latent_retain_index_list=retain_index_list, split_conds_to_windows=split_conds_to_windows) + + +class ContextWindowsExtension(ComfyExtension): + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + ContextWindowsManualNode, + WanContextWindowsManualNode, + LTXVContextWindowsNode, + ] + +def comfy_entrypoint(): + return ContextWindowsExtension() diff --git a/ComfyUI/comfy_extras/nodes_controlnet.py b/ComfyUI/comfy_extras/nodes_controlnet.py new file mode 100644 index 0000000000000000000000000000000000000000..eb476f49713eb2048af159d36db4b7119ee89a6f --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_controlnet.py @@ -0,0 +1,88 @@ +from comfy.cldm.control_types import UNION_CONTROLNET_TYPES +import nodes +import comfy.utils +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io + +class SetUnionControlNetType(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SetUnionControlNetType", + search_aliases=["set controlnet type", "union controlnet type"], + display_name="Set Union ControlNet Type", + category="model/conditioning/controlnet", + inputs=[ + io.ControlNet.Input("control_net"), + io.Combo.Input("type", options=["auto"] + list(UNION_CONTROLNET_TYPES.keys())), + ], + outputs=[ + io.ControlNet.Output(), + ], + ) + + @classmethod + def execute(cls, control_net, type) -> io.NodeOutput: + control_net = control_net.copy() + type_number = UNION_CONTROLNET_TYPES.get(type, -1) + if type_number >= 0: + control_net.set_extra_arg("control_type", [type_number]) + else: + control_net.set_extra_arg("control_type", []) + + return io.NodeOutput(control_net) + + set_controlnet_type = execute # TODO: remove + + +class ControlNetInpaintingAliMamaApply(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ControlNetInpaintingAliMamaApply", + search_aliases=["masked controlnet"], + display_name="Apply ControlNet Inpainting (AliMama)", + category="model/conditioning/controlnet", + inputs=[ + io.Conditioning.Input("positive"), + io.Conditioning.Input("negative"), + io.ControlNet.Input("control_net"), + io.Vae.Input("vae"), + io.Image.Input("image"), + io.Mask.Input("mask"), + io.Float.Input("strength", default=1.0, min=0.0, max=10.0, step=0.01), + io.Float.Input("start_percent", default=0.0, min=0.0, max=1.0, step=0.001, advanced=True), + io.Float.Input("end_percent", default=1.0, min=0.0, max=1.0, step=0.001, advanced=True), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + ], + ) + + @classmethod + def execute(cls, positive, negative, control_net, vae, image, mask, strength, start_percent, end_percent) -> io.NodeOutput: + extra_concat = [] + if control_net.concat_mask: + mask = 1.0 - mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])) + mask_apply = comfy.utils.common_upscale(mask, image.shape[2], image.shape[1], "bilinear", "center").round() + image = image * mask_apply.movedim(1, -1).repeat(1, 1, 1, image.shape[3]) + extra_concat = [mask] + + result = nodes.ControlNetApplyAdvanced().apply_controlnet(positive, negative, control_net, image, strength, start_percent, end_percent, vae=vae, extra_concat=extra_concat) + return io.NodeOutput(result[0], result[1]) + + apply_inpaint_controlnet = execute # TODO: remove + + +class ControlNetExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + SetUnionControlNetType, + ControlNetInpaintingAliMamaApply, + ] + + +async def comfy_entrypoint() -> ControlNetExtension: + return ControlNetExtension() diff --git a/ComfyUI/comfy_extras/nodes_cosmos.py b/ComfyUI/comfy_extras/nodes_cosmos.py new file mode 100644 index 0000000000000000000000000000000000000000..93cc67a6c04de97903ec58bbfacaac2385b4e3ba --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_cosmos.py @@ -0,0 +1,143 @@ +from typing_extensions import override +import nodes +import torch +import comfy.model_management +import comfy.utils +import comfy.latent_formats + +from comfy_api.latest import ComfyExtension, io + + +class EmptyCosmosLatentVideo(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="EmptyCosmosLatentVideo", + category="model/latent/cosmos", + inputs=[ + io.Int.Input("width", default=1280, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("height", default=704, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("length", default=121, min=1, max=nodes.MAX_RESOLUTION, step=8), + io.Int.Input("batch_size", default=1, min=1, max=4096), + ], + outputs=[io.Latent.Output()], + ) + + @classmethod + def execute(cls, width, height, length, batch_size=1) -> io.NodeOutput: + latent = torch.zeros([batch_size, 16, ((length - 1) // 8) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device()) + return io.NodeOutput({"samples": latent}) + + +def vae_encode_with_padding(vae, image, width, height, length, padding=0): + pixels = comfy.utils.common_upscale(image[..., :3].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1) + pixel_len = min(pixels.shape[0], length) + padded_length = min(length, (((pixel_len - 1) // 8) + 1 + padding) * 8 - 7) + padded_pixels = torch.ones((padded_length, height, width, 3)) * 0.5 + padded_pixels[:pixel_len] = pixels[:pixel_len] + latent_len = ((pixel_len - 1) // 8) + 1 + latent_temp = vae.encode(padded_pixels) + return latent_temp[:, :, :latent_len] + + +class CosmosImageToVideoLatent(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="CosmosImageToVideoLatent", + category="model/conditioning/cosmos", + inputs=[ + io.Vae.Input("vae"), + io.Int.Input("width", default=1280, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("height", default=704, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("length", default=121, min=1, max=nodes.MAX_RESOLUTION, step=8), + io.Int.Input("batch_size", default=1, min=1, max=4096), + io.Image.Input("start_image", optional=True), + io.Image.Input("end_image", optional=True), + ], + outputs=[io.Latent.Output()], + ) + + @classmethod + def execute(cls, vae, width, height, length, batch_size, start_image=None, end_image=None) -> io.NodeOutput: + latent = torch.zeros([1, 16, ((length - 1) // 8) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device()) + if start_image is None and end_image is None: + out_latent = {} + out_latent["samples"] = latent + return io.NodeOutput(out_latent) + + mask = torch.ones([latent.shape[0], 1, ((length - 1) // 8) + 1, latent.shape[-2], latent.shape[-1]], device=comfy.model_management.intermediate_device()) + + if start_image is not None: + latent_temp = vae_encode_with_padding(vae, start_image, width, height, length, padding=1) + latent[:, :, :latent_temp.shape[-3]] = latent_temp + mask[:, :, :latent_temp.shape[-3]] *= 0.0 + + if end_image is not None: + latent_temp = vae_encode_with_padding(vae, end_image, width, height, length, padding=0) + latent[:, :, -latent_temp.shape[-3]:] = latent_temp + mask[:, :, -latent_temp.shape[-3]:] *= 0.0 + + out_latent = {} + out_latent["samples"] = latent.repeat((batch_size, ) + (1,) * (latent.ndim - 1)) + out_latent["noise_mask"] = mask.repeat((batch_size, ) + (1,) * (mask.ndim - 1)) + return io.NodeOutput(out_latent) + +class CosmosPredict2ImageToVideoLatent(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="CosmosPredict2ImageToVideoLatent", + category="model/conditioning/cosmos", + inputs=[ + io.Vae.Input("vae"), + io.Int.Input("width", default=848, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("length", default=93, min=1, max=nodes.MAX_RESOLUTION, step=4), + io.Int.Input("batch_size", default=1, min=1, max=4096), + io.Image.Input("start_image", optional=True), + io.Image.Input("end_image", optional=True), + ], + outputs=[io.Latent.Output()], + ) + + @classmethod + def execute(cls, vae, width, height, length, batch_size, start_image=None, end_image=None) -> io.NodeOutput: + latent = torch.zeros([1, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device()) + if start_image is None and end_image is None: + out_latent = {} + out_latent["samples"] = latent + return io.NodeOutput(out_latent) + + mask = torch.ones([latent.shape[0], 1, ((length - 1) // 4) + 1, latent.shape[-2], latent.shape[-1]], device=comfy.model_management.intermediate_device()) + + if start_image is not None: + latent_temp = vae_encode_with_padding(vae, start_image, width, height, length, padding=1) + latent[:, :, :latent_temp.shape[-3]] = latent_temp + mask[:, :, :latent_temp.shape[-3]] *= 0.0 + + if end_image is not None: + latent_temp = vae_encode_with_padding(vae, end_image, width, height, length, padding=0) + latent[:, :, -latent_temp.shape[-3]:] = latent_temp + mask[:, :, -latent_temp.shape[-3]:] *= 0.0 + + out_latent = {} + latent_format = comfy.latent_formats.Wan21() + latent = latent_format.process_out(latent) * mask + latent * (1.0 - mask) + out_latent["samples"] = latent.repeat((batch_size, ) + (1,) * (latent.ndim - 1)) + out_latent["noise_mask"] = mask.repeat((batch_size, ) + (1,) * (mask.ndim - 1)) + return io.NodeOutput(out_latent) + + +class CosmosExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + EmptyCosmosLatentVideo, + CosmosImageToVideoLatent, + CosmosPredict2ImageToVideoLatent, + ] + + +async def comfy_entrypoint() -> CosmosExtension: + return CosmosExtension() diff --git a/ComfyUI/comfy_extras/nodes_curve.py b/ComfyUI/comfy_extras/nodes_curve.py new file mode 100644 index 0000000000000000000000000000000000000000..078d0114fd65b7352fce7b2c9129ed758b6e5c69 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_curve.py @@ -0,0 +1,91 @@ +import numpy as np + +from comfy_api.latest import ComfyExtension, io +from comfy_api.input import CurveInput +from typing_extensions import override + + +class CurveEditor(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="CurveEditor", + display_name="Curve Editor", + category="utilities", + has_intermediate_output=True, + inputs=[ + io.Curve.Input("curve"), + io.Histogram.Input("histogram", optional=True), + ], + outputs=[ + io.Curve.Output("curve"), + ], + ) + + @classmethod + def execute(cls, curve, histogram=None) -> io.NodeOutput: + result = CurveInput.from_raw(curve) + + ui = {} + if histogram is not None: + ui["histogram"] = histogram if isinstance(histogram, list) else list(histogram) + + return io.NodeOutput(result, ui=ui) if ui else io.NodeOutput(result) + + +class ImageHistogram(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ImageHistogram", + display_name="Image Histogram", + category="utilities", + inputs=[ + io.Image.Input("image"), + ], + outputs=[ + io.Histogram.Output("rgb"), + io.Histogram.Output("luminance"), + io.Histogram.Output("red"), + io.Histogram.Output("green"), + io.Histogram.Output("blue"), + ], + ) + + @classmethod + def execute(cls, image) -> io.NodeOutput: + img = image[0].cpu().numpy() + img_uint8 = np.clip(img * 255, 0, 255).astype(np.uint8) + + def bincount(data): + return np.bincount(data.ravel(), minlength=256)[:256] + + hist_r = bincount(img_uint8[:, :, 0]) + hist_g = bincount(img_uint8[:, :, 1]) + hist_b = bincount(img_uint8[:, :, 2]) + + # Average of R, G, B histograms (same as Photoshop's RGB composite) + rgb = ((hist_r + hist_g + hist_b) // 3).tolist() + + # ITU-R BT.709-6, Item 3.2 (p.6) — Derivation of luminance signal + # https://www.itu.int/rec/R-REC-BT.709-6-201506-I/en + lum = 0.2126 * img[:, :, 0] + 0.7152 * img[:, :, 1] + 0.0722 * img[:, :, 2] + luminance = bincount(np.clip(lum * 255, 0, 255).astype(np.uint8)).tolist() + + return io.NodeOutput( + rgb, + luminance, + hist_r.tolist(), + hist_g.tolist(), + hist_b.tolist(), + ) + + +class CurveExtension(ComfyExtension): + @override + async def get_node_list(self): + return [CurveEditor, ImageHistogram] + + +async def comfy_entrypoint(): + return CurveExtension() diff --git a/ComfyUI/comfy_extras/nodes_custom_sampler.py b/ComfyUI/comfy_extras/nodes_custom_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..e81b6328b7791162f80207194f983dc0d6b4dc07 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_custom_sampler.py @@ -0,0 +1,1226 @@ +import math +import comfy.samplers +import comfy.sampler_helpers +import comfy.patcher_extension +import comfy.sample +from comfy.k_diffusion import sampling as k_diffusion_sampling +from comfy.k_diffusion import sa_solver +import latent_preview +import torch +import comfy.utils +import node_helpers +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io +import re + + +class BasicScheduler(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="BasicScheduler", + category="model/sampling/schedulers", + inputs=[ + io.Model.Input("model"), + io.Combo.Input("scheduler", options=comfy.samplers.SCHEDULER_NAMES), + io.Int.Input("steps", default=20, min=1, max=10000), + io.Float.Input("denoise", default=1.0, min=0.0, max=1.0, step=0.01), + ], + outputs=[io.Sigmas.Output()] + ) + + @classmethod + def execute(cls, model, scheduler, steps, denoise) -> io.NodeOutput: + total_steps = steps + if denoise < 1.0: + if denoise <= 0.0: + return io.NodeOutput(torch.FloatTensor([])) + total_steps = int(steps/denoise) + + sigmas = comfy.samplers.calculate_sigmas(model.get_model_object("model_sampling"), scheduler, total_steps).cpu() + sigmas = sigmas[-(steps + 1):] + return io.NodeOutput(sigmas) + + get_sigmas = execute + + +class KarrasScheduler(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="KarrasScheduler", + category="model/sampling/schedulers", + inputs=[ + io.Int.Input("steps", default=20, min=1, max=10000), + io.Float.Input("sigma_max", default=14.614642, min=0.0, max=5000.0, step=0.01, round=False, advanced=True), + io.Float.Input("sigma_min", default=0.0291675, min=0.0, max=5000.0, step=0.01, round=False, advanced=True), + io.Float.Input("rho", default=7.0, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + ], + outputs=[io.Sigmas.Output()] + ) + + @classmethod + def execute(cls, steps, sigma_max, sigma_min, rho) -> io.NodeOutput: + sigmas = k_diffusion_sampling.get_sigmas_karras(n=steps, sigma_min=sigma_min, sigma_max=sigma_max, rho=rho) + return io.NodeOutput(sigmas) + + get_sigmas = execute + +class ExponentialScheduler(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ExponentialScheduler", + category="model/sampling/schedulers", + inputs=[ + io.Int.Input("steps", default=20, min=1, max=10000), + io.Float.Input("sigma_max", default=14.614642, min=0.0, max=5000.0, step=0.01, round=False, advanced=True), + io.Float.Input("sigma_min", default=0.0291675, min=0.0, max=5000.0, step=0.01, round=False, advanced=True), + ], + outputs=[io.Sigmas.Output()] + ) + + @classmethod + def execute(cls, steps, sigma_max, sigma_min) -> io.NodeOutput: + sigmas = k_diffusion_sampling.get_sigmas_exponential(n=steps, sigma_min=sigma_min, sigma_max=sigma_max) + return io.NodeOutput(sigmas) + + get_sigmas = execute + +class PolyexponentialScheduler(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="PolyexponentialScheduler", + category="model/sampling/schedulers", + inputs=[ + io.Int.Input("steps", default=20, min=1, max=10000), + io.Float.Input("sigma_max", default=14.614642, min=0.0, max=5000.0, step=0.01, round=False, advanced=True), + io.Float.Input("sigma_min", default=0.0291675, min=0.0, max=5000.0, step=0.01, round=False, advanced=True), + io.Float.Input("rho", default=1.0, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + ], + outputs=[io.Sigmas.Output()] + ) + + @classmethod + def execute(cls, steps, sigma_max, sigma_min, rho) -> io.NodeOutput: + sigmas = k_diffusion_sampling.get_sigmas_polyexponential(n=steps, sigma_min=sigma_min, sigma_max=sigma_max, rho=rho) + return io.NodeOutput(sigmas) + + get_sigmas = execute + +class LaplaceScheduler(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LaplaceScheduler", + category="model/sampling/schedulers", + inputs=[ + io.Int.Input("steps", default=20, min=1, max=10000), + io.Float.Input("sigma_max", default=14.614642, min=0.0, max=5000.0, step=0.01, round=False, advanced=True), + io.Float.Input("sigma_min", default=0.0291675, min=0.0, max=5000.0, step=0.01, round=False, advanced=True), + io.Float.Input("mu", default=0.0, min=-10.0, max=10.0, step=0.1, round=False, advanced=True), + io.Float.Input("beta", default=0.5, min=0.0, max=10.0, step=0.1, round=False, advanced=True), + ], + outputs=[io.Sigmas.Output()] + ) + + @classmethod + def execute(cls, steps, sigma_max, sigma_min, mu, beta) -> io.NodeOutput: + sigmas = k_diffusion_sampling.get_sigmas_laplace(n=steps, sigma_min=sigma_min, sigma_max=sigma_max, mu=mu, beta=beta) + return io.NodeOutput(sigmas) + + get_sigmas = execute + + +class SDTurboScheduler(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SDTurboScheduler", + category="model/sampling/schedulers", + inputs=[ + io.Model.Input("model"), + io.Int.Input("steps", default=1, min=1, max=10), + io.Float.Input("denoise", default=1.0, min=0, max=1.0, step=0.01), + ], + outputs=[io.Sigmas.Output()] + ) + + @classmethod + def execute(cls, model, steps, denoise) -> io.NodeOutput: + start_step = 10 - int(10 * denoise) + timesteps = torch.flip(torch.arange(1, 11) * 100 - 1, (0,))[start_step:start_step + steps] + sigmas = model.get_model_object("model_sampling").sigma(timesteps) + sigmas = torch.cat([sigmas, sigmas.new_zeros([1])]) + return io.NodeOutput(sigmas) + + get_sigmas = execute + +class BetaSamplingScheduler(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="BetaSamplingScheduler", + category="model/sampling/schedulers", + inputs=[ + io.Model.Input("model"), + io.Int.Input("steps", default=20, min=1, max=10000), + io.Float.Input("alpha", default=0.6, min=0.0, max=50.0, step=0.01, round=False, advanced=True), + io.Float.Input("beta", default=0.6, min=0.0, max=50.0, step=0.01, round=False, advanced=True), + ], + outputs=[io.Sigmas.Output()] + ) + + @classmethod + def execute(cls, model, steps, alpha, beta) -> io.NodeOutput: + sigmas = comfy.samplers.beta_scheduler(model.get_model_object("model_sampling"), steps, alpha=alpha, beta=beta) + return io.NodeOutput(sigmas) + + get_sigmas = execute + +class VPScheduler(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="VPScheduler", + category="model/sampling/schedulers", + inputs=[ + io.Int.Input("steps", default=20, min=1, max=10000), + io.Float.Input("beta_d", default=19.9, min=0.0, max=5000.0, step=0.01, round=False, advanced=True), #TODO: fix default values + io.Float.Input("beta_min", default=0.1, min=0.0, max=5000.0, step=0.01, round=False, advanced=True), + io.Float.Input("eps_s", default=0.001, min=0.0, max=1.0, step=0.0001, round=False, advanced=True), + ], + outputs=[io.Sigmas.Output()] + ) + + @classmethod + def execute(cls, steps, beta_d, beta_min, eps_s) -> io.NodeOutput: + sigmas = k_diffusion_sampling.get_sigmas_vp(n=steps, beta_d=beta_d, beta_min=beta_min, eps_s=eps_s) + return io.NodeOutput(sigmas) + + get_sigmas = execute + +class SplitSigmas(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SplitSigmas", + category="model/sampling/sigmas", + inputs=[ + io.Sigmas.Input("sigmas"), + io.Int.Input("step", default=0, min=0, max=10000), + ], + outputs=[ + io.Sigmas.Output(display_name="high_sigmas"), + io.Sigmas.Output(display_name="low_sigmas"), + ] + ) + + @classmethod + def execute(cls, sigmas, step) -> io.NodeOutput: + sigmas1 = sigmas[:step + 1] + sigmas2 = sigmas[step:] + return io.NodeOutput(sigmas1, sigmas2) + + get_sigmas = execute + +class SplitSigmasDenoise(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SplitSigmasDenoise", + category="model/sampling/sigmas", + inputs=[ + io.Sigmas.Input("sigmas"), + io.Float.Input("denoise", default=1.0, min=0.0, max=1.0, step=0.01), + ], + outputs=[ + io.Sigmas.Output(display_name="high_sigmas"), + io.Sigmas.Output(display_name="low_sigmas"), + ] + ) + + @classmethod + def execute(cls, sigmas, denoise) -> io.NodeOutput: + steps = max(sigmas.shape[-1] - 1, 0) + total_steps = round(steps * denoise) + sigmas1 = sigmas[:-(total_steps)] + sigmas2 = sigmas[-(total_steps + 1):] + return io.NodeOutput(sigmas1, sigmas2) + + get_sigmas = execute + +class FlipSigmas(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="FlipSigmas", + category="model/sampling/sigmas", + inputs=[io.Sigmas.Input("sigmas")], + outputs=[io.Sigmas.Output()] + ) + + @classmethod + def execute(cls, sigmas) -> io.NodeOutput: + if len(sigmas) == 0: + return io.NodeOutput(sigmas) + + sigmas = sigmas.flip(0) + if sigmas[0] == 0: + sigmas[0] = 0.0001 + return io.NodeOutput(sigmas) + + get_sigmas = execute + +class SetFirstSigma(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SetFirstSigma", + category="model/sampling/sigmas", + inputs=[ + io.Sigmas.Input("sigmas"), + io.Float.Input("sigma", default=136.0, min=0.0, max=20000.0, step=0.001, round=False), + ], + outputs=[io.Sigmas.Output()] + ) + + @classmethod + def execute(cls, sigmas, sigma) -> io.NodeOutput: + sigmas = sigmas.clone() + sigmas[0] = sigma + return io.NodeOutput(sigmas) + + set_first_sigma = execute + +class ExtendIntermediateSigmas(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ExtendIntermediateSigmas", + search_aliases=["interpolate sigmas"], + category="model/sampling/sigmas", + inputs=[ + io.Sigmas.Input("sigmas"), + io.Int.Input("steps", default=2, min=1, max=100), + io.Float.Input("start_at_sigma", default=-1.0, min=-1.0, max=20000.0, step=0.01, round=False), + io.Float.Input("end_at_sigma", default=12.0, min=0.0, max=20000.0, step=0.01, round=False), + io.Combo.Input("spacing", options=['linear', 'cosine', 'sine']), + ], + outputs=[io.Sigmas.Output()] + ) + + @classmethod + def execute(cls, sigmas: torch.Tensor, steps: int, start_at_sigma: float, end_at_sigma: float, spacing: str) -> io.NodeOutput: + if start_at_sigma < 0: + start_at_sigma = float("inf") + + interpolator = { + 'linear': lambda x: x, + 'cosine': lambda x: torch.sin(x*math.pi/2), + 'sine': lambda x: 1 - torch.cos(x*math.pi/2) + }[spacing] + + # linear space for our interpolation function + x = torch.linspace(0, 1, steps + 1, device=sigmas.device)[1:-1] + computed_spacing = interpolator(x) + + extended_sigmas = [] + for i in range(len(sigmas) - 1): + sigma_current = sigmas[i] + sigma_next = sigmas[i+1] + + extended_sigmas.append(sigma_current) + + if end_at_sigma <= sigma_current <= start_at_sigma: + interpolated_steps = computed_spacing * (sigma_next - sigma_current) + sigma_current + extended_sigmas.extend(interpolated_steps.tolist()) + + # Add the last sigma value + if len(sigmas) > 0: + extended_sigmas.append(sigmas[-1]) + + extended_sigmas = torch.FloatTensor(extended_sigmas) + + return io.NodeOutput(extended_sigmas) + + extend = execute + + +class SamplingPercentToSigma(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SamplingPercentToSigma", + category="model/sampling/sigmas", + inputs=[ + io.Model.Input("model"), + io.Float.Input("sampling_percent", default=0.0, min=0.0, max=1.0, step=0.0001), + io.Boolean.Input("return_actual_sigma", default=False, tooltip="Return the actual sigma value instead of the value used for interval checks.\nThis only affects results at 0.0 and 1.0."), + ], + outputs=[io.Float.Output(display_name="sigma_value")] + ) + + @classmethod + def execute(cls, model, sampling_percent, return_actual_sigma) -> io.NodeOutput: + model_sampling = model.get_model_object("model_sampling") + sigma_val = model_sampling.percent_to_sigma(sampling_percent) + if return_actual_sigma: + if sampling_percent == 0.0: + sigma_val = model_sampling.sigma_max.item() + elif sampling_percent == 1.0: + sigma_val = model_sampling.sigma_min.item() + return io.NodeOutput(sigma_val) + + get_sigma = execute + + +class KSamplerSelect(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="KSamplerSelect", + category="model/sampling/samplers", + inputs=[io.Combo.Input("sampler_name", options=comfy.samplers.SAMPLER_NAMES)], + outputs=[io.Sampler.Output()] + ) + + @classmethod + def execute(cls, sampler_name) -> io.NodeOutput: + sampler = comfy.samplers.sampler_object(sampler_name) + return io.NodeOutput(sampler) + + get_sampler = execute + +class SamplerDPMPP_3M_SDE(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SamplerDPMPP_3M_SDE", + category="model/sampling/samplers", + inputs=[ + io.Float.Input("eta", default=1.0, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + io.Float.Input("s_noise", default=1.0, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + io.Combo.Input("noise_device", options=['gpu', 'cpu'], advanced=True), + ], + outputs=[io.Sampler.Output()] + ) + + @classmethod + def execute(cls, eta, s_noise, noise_device) -> io.NodeOutput: + if noise_device == 'cpu': + sampler_name = "dpmpp_3m_sde" + else: + sampler_name = "dpmpp_3m_sde_gpu" + sampler = comfy.samplers.ksampler(sampler_name, {"eta": eta, "s_noise": s_noise}) + return io.NodeOutput(sampler) + + get_sampler = execute + +class SamplerDPMPP_2M_SDE(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SamplerDPMPP_2M_SDE", + category="model/sampling/samplers", + inputs=[ + io.Combo.Input("solver_type", options=['midpoint', 'heun']), + io.Float.Input("eta", default=1.0, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + io.Float.Input("s_noise", default=1.0, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + io.Combo.Input("noise_device", options=['gpu', 'cpu'], advanced=True), + ], + outputs=[io.Sampler.Output()] + ) + + @classmethod + def execute(cls, solver_type, eta, s_noise, noise_device) -> io.NodeOutput: + if noise_device == 'cpu': + sampler_name = "dpmpp_2m_sde" + else: + sampler_name = "dpmpp_2m_sde_gpu" + sampler = comfy.samplers.ksampler(sampler_name, {"eta": eta, "s_noise": s_noise, "solver_type": solver_type}) + return io.NodeOutput(sampler) + + get_sampler = execute + + +class SamplerDPMPP_SDE(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SamplerDPMPP_SDE", + category="model/sampling/samplers", + inputs=[ + io.Float.Input("eta", default=1.0, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + io.Float.Input("s_noise", default=1.0, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + io.Float.Input("r", default=0.5, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + io.Combo.Input("noise_device", options=['gpu', 'cpu'], advanced=True), + ], + outputs=[io.Sampler.Output()] + ) + + @classmethod + def execute(cls, eta, s_noise, r, noise_device) -> io.NodeOutput: + if noise_device == 'cpu': + sampler_name = "dpmpp_sde" + else: + sampler_name = "dpmpp_sde_gpu" + sampler = comfy.samplers.ksampler(sampler_name, {"eta": eta, "s_noise": s_noise, "r": r}) + return io.NodeOutput(sampler) + + get_sampler = execute + +class SamplerDPMPP_2S_Ancestral(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SamplerDPMPP_2S_Ancestral", + category="model/sampling/samplers", + inputs=[ + io.Float.Input("eta", default=1.0, min=0.0, max=100.0, step=0.01, round=False), + io.Float.Input("s_noise", default=1.0, min=0.0, max=100.0, step=0.01, round=False), + ], + outputs=[io.Sampler.Output()] + ) + + @classmethod + def execute(cls, eta, s_noise) -> io.NodeOutput: + sampler = comfy.samplers.ksampler("dpmpp_2s_ancestral", {"eta": eta, "s_noise": s_noise}) + return io.NodeOutput(sampler) + + get_sampler = execute + +class SamplerEulerAncestral(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SamplerEulerAncestral", + category="model/sampling/samplers", + inputs=[ + io.Float.Input("eta", default=1.0, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + io.Float.Input("s_noise", default=1.0, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + ], + outputs=[io.Sampler.Output()] + ) + + @classmethod + def execute(cls, eta, s_noise) -> io.NodeOutput: + sampler = comfy.samplers.ksampler("euler_ancestral", {"eta": eta, "s_noise": s_noise}) + return io.NodeOutput(sampler) + + get_sampler = execute + +class SamplerEulerAncestralCFGPP(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SamplerEulerAncestralCFGPP", + display_name="SamplerEulerAncestralCFG++", + category="model/sampling/samplers", + inputs=[ + io.Float.Input("eta", default=1.0, min=0.0, max=1.0, step=0.01, round=False), + io.Float.Input("s_noise", default=1.0, min=0.0, max=10.0, step=0.01, round=False), + ], + outputs=[io.Sampler.Output()] + ) + + @classmethod + def execute(cls, eta, s_noise) -> io.NodeOutput: + sampler = comfy.samplers.ksampler( + "euler_ancestral_cfg_pp", + {"eta": eta, "s_noise": s_noise}) + return io.NodeOutput(sampler) + + get_sampler = execute + +class SamplerLMS(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SamplerLMS", + category="model/sampling/samplers", + inputs=[io.Int.Input("order", default=4, min=1, max=100, advanced=True)], + outputs=[io.Sampler.Output()] + ) + + @classmethod + def execute(cls, order) -> io.NodeOutput: + sampler = comfy.samplers.ksampler("lms", {"order": order}) + return io.NodeOutput(sampler) + + get_sampler = execute + +class SamplerDPMAdaptative(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SamplerDPMAdaptative", + category="model/sampling/samplers", + inputs=[ + io.Int.Input("order", default=3, min=2, max=3, advanced=True), + io.Float.Input("rtol", default=0.05, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + io.Float.Input("atol", default=0.0078, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + io.Float.Input("h_init", default=0.05, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + io.Float.Input("pcoeff", default=0.0, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + io.Float.Input("icoeff", default=1.0, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + io.Float.Input("dcoeff", default=0.0, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + io.Float.Input("accept_safety", default=0.81, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + io.Float.Input("eta", default=0.0, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + io.Float.Input("s_noise", default=1.0, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + ], + outputs=[io.Sampler.Output()] + ) + + @classmethod + def execute(cls, order, rtol, atol, h_init, pcoeff, icoeff, dcoeff, accept_safety, eta, s_noise) -> io.NodeOutput: + sampler = comfy.samplers.ksampler("dpm_adaptive", {"order": order, "rtol": rtol, "atol": atol, "h_init": h_init, "pcoeff": pcoeff, + "icoeff": icoeff, "dcoeff": dcoeff, "accept_safety": accept_safety, "eta": eta, + "s_noise":s_noise }) + return io.NodeOutput(sampler) + + get_sampler = execute + + +class SamplerER_SDE(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SamplerER_SDE", + category="model/sampling/samplers", + inputs=[ + io.Combo.Input("solver_type", options=["ER-SDE", "Reverse-time SDE", "ODE"]), + io.Int.Input("max_stage", default=3, min=1, max=3, advanced=True), + io.Float.Input("eta", default=1.0, min=0.0, max=100.0, step=0.01, round=False, tooltip="Stochastic strength of reverse-time SDE.\nWhen eta=0, it reduces to deterministic ODE. This setting doesn't apply to ER-SDE solver type.", advanced=True), + io.Float.Input("s_noise", default=1.0, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + ], + outputs=[io.Sampler.Output()] + ) + + @classmethod + def execute(cls, solver_type, max_stage, eta, s_noise) -> io.NodeOutput: + if solver_type == "ODE" or (solver_type == "Reverse-time SDE" and eta == 0): + eta = 0 + s_noise = 0 + + def reverse_time_sde_noise_scaler(x): + return x ** (eta + 1) + + if solver_type == "ER-SDE": + # Use the default one in sample_er_sde() + noise_scaler = None + else: + noise_scaler = reverse_time_sde_noise_scaler + + sampler_name = "er_sde" + sampler = comfy.samplers.ksampler(sampler_name, {"s_noise": s_noise, "noise_scaler": noise_scaler, "max_stage": max_stage}) + return io.NodeOutput(sampler) + + get_sampler = execute + + +class SamplerSASolver(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SamplerSASolver", + search_aliases=["sde"], + category="model/sampling/samplers", + inputs=[ + io.Model.Input("model"), + io.Float.Input("eta", default=1.0, min=0.0, max=10.0, step=0.01, round=False, advanced=True), + io.Float.Input("sde_start_percent", default=0.2, min=0.0, max=1.0, step=0.001, advanced=True), + io.Float.Input("sde_end_percent", default=0.8, min=0.0, max=1.0, step=0.001, advanced=True), + io.Float.Input("s_noise", default=1.0, min=0.0, max=100.0, step=0.01, round=False, advanced=True), + io.Int.Input("predictor_order", default=3, min=1, max=6, advanced=True), + io.Int.Input("corrector_order", default=4, min=0, max=6, advanced=True), + io.Boolean.Input("use_pece", advanced=True), + io.Boolean.Input("simple_order_2", advanced=True), + ], + outputs=[io.Sampler.Output()] + ) + + @classmethod + def execute(cls, model, eta, sde_start_percent, sde_end_percent, s_noise, predictor_order, corrector_order, use_pece, simple_order_2) -> io.NodeOutput: + model_sampling = model.get_model_object("model_sampling") + start_sigma = model_sampling.percent_to_sigma(sde_start_percent) + end_sigma = model_sampling.percent_to_sigma(sde_end_percent) + tau_func = sa_solver.get_tau_interval_func(start_sigma, end_sigma, eta=eta) + + sampler_name = "sa_solver" + sampler = comfy.samplers.ksampler( + sampler_name, + { + "tau_func": tau_func, + "s_noise": s_noise, + "predictor_order": predictor_order, + "corrector_order": corrector_order, + "use_pece": use_pece, + "simple_order_2": simple_order_2, + }, + ) + return io.NodeOutput(sampler) + + get_sampler = execute + + +class SamplerSEEDS2(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SamplerSEEDS2", + search_aliases=["sde", "exp heun"], + category="model/sampling/samplers", + inputs=[ + io.Combo.Input("solver_type", options=["phi_1", "phi_2"]), + io.Float.Input("eta", default=1.0, min=0.0, max=100.0, step=0.01, round=False, tooltip="Stochastic strength", advanced=True), + io.Float.Input("s_noise", default=1.0, min=0.0, max=100.0, step=0.01, round=False, tooltip="SDE noise multiplier", advanced=True), + io.Float.Input("r", default=0.5, min=0.01, max=1.0, step=0.01, round=False, tooltip="Relative step size for the intermediate stage (c2 node)", advanced=True), + ], + outputs=[io.Sampler.Output()], + description=( + "This sampler node can represent multiple samplers:\n\n" + "seeds_2\n" + "- default setting\n\n" + "exp_heun_2_x0\n" + "- solver_type=phi_2, r=1.0, eta=0.0\n\n" + "exp_heun_2_x0_sde\n" + "- solver_type=phi_2, r=1.0, eta=1.0, s_noise=1.0" + ) + ) + + @classmethod + def execute(cls, solver_type, eta, s_noise, r) -> io.NodeOutput: + sampler_name = "seeds_2" + sampler = comfy.samplers.ksampler( + sampler_name, + {"eta": eta, "s_noise": s_noise, "r": r, "solver_type": solver_type}, + ) + return io.NodeOutput(sampler) + + +class Noise_EmptyNoise: + def __init__(self): + self.seed = 0 + + def generate_noise(self, input_latent): + latent_image = input_latent["samples"] + if latent_image.is_nested: + tensors = latent_image.unbind() + zeros = [] + for t in tensors: + zeros.append(torch.zeros(t.shape, dtype=t.dtype, layout=t.layout, device="cpu")) + return comfy.nested_tensor.NestedTensor(zeros) + else: + return torch.zeros(latent_image.shape, dtype=latent_image.dtype, layout=latent_image.layout, device="cpu") + + +class Noise_RandomNoise: + def __init__(self, seed): + self.seed = seed + + def generate_noise(self, input_latent): + latent_image = input_latent["samples"] + batch_inds = input_latent["batch_index"] if "batch_index" in input_latent else None + return comfy.sample.prepare_noise(latent_image, self.seed, batch_inds) + +class SamplerCustom(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SamplerCustom", + category="model/sampling/custom", + inputs=[ + io.Model.Input("model"), + io.Boolean.Input("add_noise", default=True, advanced=True), + io.Int.Input("noise_seed", default=0, min=0, max=0xffffffffffffffff, control_after_generate=True), + io.Float.Input("cfg", default=8.0, min=0.0, max=100.0, step=0.1, round=0.01), + io.Conditioning.Input("positive"), + io.Conditioning.Input("negative"), + io.Sampler.Input("sampler"), + io.Sigmas.Input("sigmas"), + io.Latent.Input("latent_image"), + ], + outputs=[ + io.Latent.Output(display_name="output"), + io.Latent.Output(display_name="denoised_output"), + ] + ) + + @classmethod + def execute(cls, model, add_noise, noise_seed, cfg, positive, negative, sampler, sigmas, latent_image) -> io.NodeOutput: + latent = latent_image + latent_image = latent["samples"] + latent = latent.copy() + latent_image = comfy.sample.fix_empty_latent_channels(model, latent_image, latent.get("downscale_ratio_spacial", None), latent.get("downscale_ratio_temporal", None)) + latent["samples"] = latent_image + + if not add_noise: + noise = Noise_EmptyNoise().generate_noise(latent) + else: + noise = Noise_RandomNoise(noise_seed).generate_noise(latent) + + noise_mask = None + if "noise_mask" in latent: + noise_mask = latent["noise_mask"] + + x0_output = {} + callback = latent_preview.prepare_callback(model, sigmas.shape[-1] - 1, x0_output) + + disable_pbar = not comfy.utils.PROGRESS_BAR_ENABLED + samples = comfy.sample.sample_custom(model, noise, cfg, sampler, sigmas, positive, negative, latent_image, noise_mask=noise_mask, callback=callback, disable_pbar=disable_pbar, seed=noise_seed) + + out = latent.copy() + out.pop("downscale_ratio_spacial", None) + out.pop("downscale_ratio_temporal", None) + out["samples"] = samples + if "x0" in x0_output: + x0 = x0_output["x0"] + if samples.is_nested and not x0.is_nested: + latent_shapes = [x.shape for x in samples.unbind()] + x0 = comfy.nested_tensor.NestedTensor(comfy.utils.unpack_latents(x0, latent_shapes)) + x0_out = model.model.process_latent_out(x0.cpu()) + out_denoised = latent.copy() + out_denoised["samples"] = x0_out + else: + out_denoised = out + return io.NodeOutput(out, out_denoised) + + sample = execute + +class Guider_Basic(comfy.samplers.CFGGuider): + def set_conds(self, positive): + self.inner_set_conds({"positive": positive}) + +class BasicGuider(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="BasicGuider", + display_name="Basic Guider", + category="model/sampling/guiders", + inputs=[ + io.Model.Input("model"), + io.Conditioning.Input("conditioning"), + ], + outputs=[io.Guider.Output()] + ) + + @classmethod + def execute(cls, model, conditioning) -> io.NodeOutput: + guider = Guider_Basic(model) + guider.set_conds(conditioning) + return io.NodeOutput(guider) + + get_guider = execute + +class CFGGuider(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="CFGGuider", + display_name="CFG Guider", + category="model/sampling/guiders", + inputs=[ + io.Model.Input("model"), + io.Conditioning.Input("positive"), + io.Conditioning.Input("negative"), + io.Float.Input("cfg", default=8.0, min=0.0, max=100.0, step=0.1, round=0.01), + ], + outputs=[io.Guider.Output()] + ) + + @classmethod + def execute(cls, model, positive, negative, cfg) -> io.NodeOutput: + guider = comfy.samplers.CFGGuider(model) + guider.set_conds(positive, negative) + guider.set_cfg(cfg) + return io.NodeOutput(guider) + + get_guider = execute + +class Guider_DualCFG(comfy.samplers.CFGGuider): + def set_cfg(self, cfg1, cfg2, nested=False): + self.cfg1 = cfg1 + self.cfg2 = cfg2 + self.nested = nested + + def set_conds(self, positive, middle, negative): + middle = node_helpers.conditioning_set_values(middle, {"prompt_type": "negative"}) + self.inner_set_conds({"positive": positive, "middle": middle, "negative": negative}) + + def predict_noise(self, x, timestep, model_options={}, seed=None): + negative_cond = self.conds.get("negative", None) + middle_cond = self.conds.get("middle", None) + positive_cond = self.conds.get("positive", None) + + if self.nested: + out = comfy.samplers.calc_cond_batch(self.inner_model, [negative_cond, middle_cond, positive_cond], x, timestep, model_options) + pred_text = comfy.samplers.cfg_function(self.inner_model, out[2], out[1], self.cfg1, x, timestep, model_options=model_options, cond=positive_cond, uncond=middle_cond) + return out[0] + self.cfg2 * (pred_text - out[0]) + else: + if model_options.get("disable_cfg1_optimization", False) == False: + if math.isclose(self.cfg2, 1.0): + negative_cond = None + if math.isclose(self.cfg1, 1.0): + middle_cond = None + + out = comfy.samplers.calc_cond_batch(self.inner_model, [negative_cond, middle_cond, positive_cond], x, timestep, model_options) + return comfy.samplers.cfg_function(self.inner_model, out[1], out[0], self.cfg2, x, timestep, model_options=model_options, cond=middle_cond, uncond=negative_cond) + (out[2] - out[1]) * self.cfg1 + +class DualCFGGuider(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="DualCFGGuider", + search_aliases=["dual prompt guidance"], + display_name="Dual CFG Guider", + category="model/sampling/guiders", + inputs=[ + io.Model.Input("model"), + io.Conditioning.Input("cond1"), + io.Conditioning.Input("cond2"), + io.Conditioning.Input("negative"), + io.Float.Input("cfg_conds", default=8.0, min=0.0, max=100.0, step=0.1, round=0.01), + io.Float.Input("cfg_cond2_negative", default=8.0, min=0.0, max=100.0, step=0.1, round=0.01), + io.Combo.Input("style", options=["regular", "nested"]), + ], + outputs=[io.Guider.Output()] + ) + + @classmethod + def execute(cls, model, cond1, cond2, negative, cfg_conds, cfg_cond2_negative, style) -> io.NodeOutput: + guider = Guider_DualCFG(model) + guider.set_conds(cond1, cond2, negative) + guider.set_cfg(cfg_conds, cfg_cond2_negative, nested=(style == "nested")) + return io.NodeOutput(guider) + + get_guider = execute + +class Guider_DualModel(comfy.samplers.CFGGuider): + # Runs the positive (cond) pass on the main model and the negative (uncond) pass on a separate model + def __init__(self, model_patcher, uncond_model_patcher): + super().__init__(model_patcher) + self.uncond_model_patcher = uncond_model_patcher + self.uncond_inner = None + + def outer_sample(self, noise, latent_image, sampler, sigmas, denoise_mask=None, callback=None, disable_pbar=False, seed=None, latent_shapes=None): + self.uncond_inner = None + self.uncond_loaded = [] + self._uncond_neg = None + # skip at cfg 1.0 + if not math.isclose(self.cfg, 1.0): + uc = {"negative": list(map(lambda a: a.copy(), self.conds["negative"]))} + self.uncond_inner, uc, self.uncond_loaded = comfy.sampler_helpers.prepare_sampling( + self.uncond_model_patcher, noise.shape, uc, self.uncond_model_patcher.model_options) + self._uncond_neg = uc["negative"] + self.uncond_model_patcher.pre_run() + try: + return super().outer_sample(noise, latent_image, sampler, sigmas, denoise_mask, callback, disable_pbar, seed, latent_shapes=latent_shapes) + finally: + if self.uncond_inner is not None: + self.uncond_model_patcher.cleanup() + comfy.sampler_helpers.cleanup_models({"negative": self._uncond_neg}, self.uncond_loaded) + self.uncond_inner = None + + def inner_sample(self, noise, latent_image, device, sampler, sigmas, denoise_mask, callback, disable_pbar, seed, latent_shapes=None): + if self.uncond_inner is not None: + li = latent_image + if li is not None and torch.count_nonzero(li) > 0: + li = self.uncond_inner.process_latent_in(li) + self._uncond_conds = comfy.samplers.process_conds( + self.uncond_inner, noise, {"negative": self._uncond_neg}, device, li, denoise_mask, seed, latent_shapes=latent_shapes)["negative"] + return super().inner_sample(noise, latent_image, device, sampler, sigmas, denoise_mask, callback, disable_pbar, seed, latent_shapes=latent_shapes) + + def predict_noise(self, x, timestep, model_options={}, seed=None): + positive = self.conds.get("positive", None) + cond = comfy.samplers.calc_cond_batch(self.inner_model, [positive], x, timestep, model_options)[0] + # uncond model not loaded (base cfg==1/no negative), or cfg driven to 1.0 this step -> single model, cond only + if self.uncond_inner is None or (math.isclose(self.cfg, 1.0) and not model_options.get("disable_cfg1_optimization", False)): + return cond + + uncond_model_options = model_options + if "multigpu_clones" in model_options: # TODO: support multigpu instead of just running uncond on a single GPU + uncond_model_options = {k: v for k, v in model_options.items() if k != "multigpu_clones"} + uncond = comfy.samplers.calc_cond_batch(self.uncond_inner, [self._uncond_conds], x, timestep, uncond_model_options)[0] + return comfy.samplers.cfg_function(self.inner_model, cond, uncond, self.cfg, x, timestep, + model_options=model_options, cond=positive, uncond=self._uncond_conds) + +class DualModelGuider(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="DualModelGuider", + display_name="Dual Model CFG Guider", + category="model/sampling/guiders", + is_experimental=True, + inputs=[ + io.Model.Input("model", tooltip="Model used for the positive (conditional) pass."), + io.Model.Input("model_negative", optional=True, tooltip="Model used for the negative (unconditional) pass. Use the same model for ordinary CFG."), + io.Conditioning.Input("positive"), + io.Float.Input("cfg", default=4.0, min=0.0, max=100.0, step=0.1, round=0.01), + io.Conditioning.Input("negative", optional=True, tooltip="Negative conditioning run on the negative model. Leave unconnected for a text-free (image-only) unconditional pass."), + ], + outputs=[io.Guider.Output()], + ) + + @classmethod + def execute(cls, model, positive, cfg, model_negative=None, negative=None) -> io.NodeOutput: + if negative is None: + negative = [[None, {}]] # null cond -> no cross_attn -> model runs image-only + + guider = Guider_DualModel(model, model_negative) if model_negative is not None else comfy.samplers.CFGGuider(model) + guider.set_conds(positive, negative) + guider.set_cfg(cfg) + return io.NodeOutput(guider) + + get_guider = execute + +class DisableNoise(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="DisableNoise", + search_aliases=["zero noise"], + category="model/sampling/noise", + inputs=[], + outputs=[io.Noise.Output()] + ) + + @classmethod + def execute(cls) -> io.NodeOutput: + return io.NodeOutput(Noise_EmptyNoise()) + + get_noise = execute + + +class RandomNoise(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="RandomNoise", + category="model/sampling/noise", + inputs=[io.Int.Input("noise_seed", default=0, min=0, max=0xffffffffffffffff, control_after_generate=True)], + outputs=[io.Noise.Output()] + ) + + @classmethod + def execute(cls, noise_seed) -> io.NodeOutput: + return io.NodeOutput(Noise_RandomNoise(noise_seed)) + + get_noise = execute + + +class SamplerCustomAdvanced(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SamplerCustomAdvanced", + category="model/sampling/custom", + inputs=[ + io.Noise.Input("noise"), + io.Guider.Input("guider"), + io.Sampler.Input("sampler"), + io.Sigmas.Input("sigmas"), + io.Latent.Input("latent_image"), + ], + outputs=[ + io.Latent.Output(display_name="output"), + io.Latent.Output(display_name="denoised_output"), + ] + ) + + @classmethod + def execute(cls, noise, guider, sampler, sigmas, latent_image) -> io.NodeOutput: + latent = latent_image + latent_image = latent["samples"] + latent = latent.copy() + latent_image = comfy.sample.fix_empty_latent_channels(guider.model_patcher, latent_image, latent.get("downscale_ratio_spacial", None), latent.get("downscale_ratio_temporal", None)) + latent["samples"] = latent_image + + noise_mask = None + if "noise_mask" in latent: + noise_mask = latent["noise_mask"] + + x0_output = {} + callback = latent_preview.prepare_callback(guider.model_patcher, sigmas.shape[-1] - 1, x0_output) + + disable_pbar = not comfy.utils.PROGRESS_BAR_ENABLED + samples = guider.sample(noise.generate_noise(latent), latent_image, sampler, sigmas, denoise_mask=noise_mask, callback=callback, disable_pbar=disable_pbar, seed=noise.seed) + samples = samples.to(comfy.model_management.intermediate_device()) + + out = latent.copy() + out.pop("downscale_ratio_spacial", None) + out.pop("downscale_ratio_temporal", None) + out["samples"] = samples + if "x0" in x0_output: + x0 = x0_output["x0"] + if samples.is_nested and not x0.is_nested: + latent_shapes = [x.shape for x in samples.unbind()] + x0 = comfy.nested_tensor.NestedTensor(comfy.utils.unpack_latents(x0, latent_shapes)) + x0_out = guider.model_patcher.model.process_latent_out(x0.cpu()) + out_denoised = latent.copy() + out_denoised["samples"] = x0_out + else: + out_denoised = out + return io.NodeOutput(out, out_denoised) + + sample = execute + +class AddNoise(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="AddNoise", + category="model/sampling/noise", + is_experimental=True, + inputs=[ + io.Model.Input("model"), + io.Noise.Input("noise"), + io.Sigmas.Input("sigmas"), + io.Latent.Input("latent_image"), + ], + outputs=[ + io.Latent.Output(), + ] + ) + + @classmethod + def execute(cls, model, noise, sigmas, latent_image) -> io.NodeOutput: + if len(sigmas) == 0: + return io.NodeOutput(latent_image) + + latent = latent_image + latent_image = latent["samples"] + + noisy = noise.generate_noise(latent) + + model_sampling = model.get_model_object("model_sampling") + process_latent_out = model.get_model_object("process_latent_out") + process_latent_in = model.get_model_object("process_latent_in") + + if len(sigmas) > 1: + scale = torch.abs(sigmas[0] - sigmas[-1]) + else: + scale = sigmas[0] + + if torch.count_nonzero(latent_image) > 0: #Don't shift the empty latent image. + latent_image = process_latent_in(latent_image) + noisy = model_sampling.noise_scaling(scale, noisy, latent_image) + noisy = process_latent_out(noisy) + noisy = torch.nan_to_num(noisy, nan=0.0, posinf=0.0, neginf=0.0) + + out = latent.copy() + out["samples"] = noisy + return io.NodeOutput(out) + + add_noise = execute + +class ManualSigmas(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ManualSigmas", + search_aliases=["custom noise schedule", "define sigmas"], + category="model/sampling/sigmas", + is_experimental=True, + inputs=[ + io.String.Input("sigmas", default="1, 0.5", multiline=False) + ], + outputs=[io.Sigmas.Output()] + ) + + @classmethod + def execute(cls, sigmas) -> io.NodeOutput: + sigmas = re.findall(r"[-+]?(?:\d*\.*\d+)", sigmas) + sigmas = [float(i) for i in sigmas] + sigmas = torch.FloatTensor(sigmas) + return io.NodeOutput(sigmas) + +class CFGOverride(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="CFGOverride", + display_name="CFG Override", + description="Override cfg to a fixed value over a [start, end] percent (sigma) range. " + "With multiple overrides, the one nearest the sampler wins on overlap.", + category="model/sampling/guiders", + inputs=[ + io.Model.Input("model"), + io.Float.Input("cfg", default=1.0, min=0.0, max=100.0, step=0.1, round=0.01), + io.Float.Input("start_percent", default=0.0, min=0.0, max=1.0, step=0.001), + io.Float.Input("end_percent", default=1.0, min=0.0, max=1.0, step=0.001), + ], + outputs=[io.Model.Output()], + ) + + @classmethod + def execute(cls, model, cfg, start_percent, end_percent) -> io.NodeOutput: + ms = model.get_model_object("model_sampling") + sigma_hi = ms.percent_to_sigma(start_percent) # percent->sigma decreasing, so hi >= lo + sigma_lo = ms.percent_to_sigma(end_percent) + + def predict_noise_wrapper(executor, *args, **kwargs): + sigma = float(args[1].flatten()[0]) # args = (x, timestep, model_options, seed) + if not (sigma_lo <= sigma <= sigma_hi): + return executor(*args, **kwargs) + guider = executor.class_obj # guider.cfg feeds cond_scale + saved = guider.cfg + guider.cfg = cfg + try: + return executor(*args, **kwargs) + finally: + guider.cfg = saved # restore for other steps/overrides + + m = model.clone() + m.add_wrapper(comfy.patcher_extension.WrappersMP.PREDICT_NOISE, predict_noise_wrapper) + return io.NodeOutput(m) + + +class CustomSamplersExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + SamplerCustom, + CFGOverride, + BasicScheduler, + KarrasScheduler, + ExponentialScheduler, + PolyexponentialScheduler, + LaplaceScheduler, + VPScheduler, + BetaSamplingScheduler, + SDTurboScheduler, + KSamplerSelect, + SamplerEulerAncestral, + SamplerEulerAncestralCFGPP, + SamplerLMS, + SamplerDPMPP_3M_SDE, + SamplerDPMPP_2M_SDE, + SamplerDPMPP_SDE, + SamplerDPMPP_2S_Ancestral, + SamplerDPMAdaptative, + SamplerER_SDE, + SamplerSASolver, + SamplerSEEDS2, + SplitSigmas, + SplitSigmasDenoise, + FlipSigmas, + SetFirstSigma, + ExtendIntermediateSigmas, + SamplingPercentToSigma, + CFGGuider, + DualCFGGuider, + DualModelGuider, + BasicGuider, + RandomNoise, + DisableNoise, + AddNoise, + SamplerCustomAdvanced, + ManualSigmas, + ] + + +async def comfy_entrypoint() -> CustomSamplersExtension: + return CustomSamplersExtension() diff --git a/ComfyUI/comfy_extras/nodes_dataset.py b/ComfyUI/comfy_extras/nodes_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..71e5ee368632f56f9d752943a4988c475a0b621c --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_dataset.py @@ -0,0 +1,2159 @@ +import logging +import os +import json + +import av +import numpy as np +import torch +from PIL import Image +from typing_extensions import override + +import folder_paths +import node_helpers +from comfy_api.latest import ComfyExtension, io, Input, InputImpl, Types + + +def load_and_process_images(image_files, input_dir): + """Utility function to load and process a list of images. + + Args: + image_files: List of image filenames + input_dir: Base directory containing the images + resize_method: How to handle images of different sizes ("None", "Stretch", "Crop", "Pad") + + Returns: + torch.Tensor: Batch of processed images + """ + if not image_files: + raise ValueError("No valid images found in input") + + output_images = [] + + for file in image_files: + image_path = os.path.join(input_dir, file) + img = node_helpers.pillow(Image.open, image_path) + + if img.mode == "I": + img = img.point(lambda i: i * (1 / 255)) + img = img.convert("RGB") + img_array = np.array(img).astype(np.float32) / 255.0 + img_tensor = torch.from_numpy(img_array)[None,] + output_images.append(img_tensor) + + return output_images + + +def secure_subfolder_path(base_dir, folder_name): + """Resolve folder_name inside base_dir, rejecting anything that escapes it. + + Blocks '..', absolute paths, drive letters and symlink escapes using the + same realpath containment check as the core file endpoints. + """ + target = os.path.abspath(os.path.join(base_dir, folder_name)) + if not folder_paths.is_within_directory(base_dir, target): + raise ValueError(f"Invalid folder name {folder_name!r}: resolves outside of {base_dir}") + return target + + +def list_dataset_folders(): + """Relative paths of dataset folders found under all dataset roots. + + Any subfolder containing a metadata.json or *.safetensors shard counts as + a dataset; the walk doesn't descend into matched folders. + + Symlinked directories are followed, but symlink loops are avoided. + """ + found = set() + + for root in folder_paths.get_folder_paths("datasets"): + if not os.path.isdir(root): + continue + + root = os.path.abspath(root) + seen_dirs = set() + + for dirpath, subdirs, filenames in os.walk(root, followlinks=True): + try: + st = os.stat(dirpath) # follows symlinks + except OSError: + subdirs[:] = [] + continue + + dir_key = (st.st_dev, st.st_ino) + if dir_key in seen_dirs: + subdirs[:] = [] + continue + + seen_dirs.add(dir_key) + + if dirpath != root and ( + "metadata.json" in filenames + or any(f.endswith(".safetensors") for f in filenames) + ): + found.add(os.path.relpath(dirpath, root).replace(os.sep, "/")) + subdirs[:] = [] + continue + + kept_subdirs = [] + for name in subdirs: + child = os.path.join(dirpath, name) + try: + child_st = os.stat(child) # follows symlinks + except OSError: + continue + + child_key = (child_st.st_dev, child_st.st_ino) + if child_key not in seen_dirs: + kept_subdirs.append(name) + + subdirs[:] = kept_subdirs + + return sorted(found) + + +def get_dataset_save_dir(folder_name): + """Resolve the folder to save a new dataset into, inside the default root. + + The folder is not created here; callers makedirs after validation. + """ + root = folder_paths.get_folder_paths("datasets")[0] + target = secure_subfolder_path(root, folder_name) + if os.path.realpath(target) == os.path.realpath(root): + raise ValueError("folder_name must name a subfolder of the datasets directory, e.g. 'my_dataset'.") + return target + + +def get_dataset_dir(folder_name): + """Find an existing dataset folder by relative name across all dataset roots.""" + roots = folder_paths.get_folder_paths("datasets") + for root in roots: + target = secure_subfolder_path(root, folder_name) + if os.path.realpath(target) == os.path.realpath(root): + raise ValueError("folder_name must name a subfolder of the datasets directory, e.g. 'my_dataset'.") + if os.path.isdir(target): + return target + raise ValueError(f"Dataset folder {folder_name!r} not found in: {', '.join(roots)}") + + +VALID_VIDEO_EXTENSIONS = [".mp4", ".avi", ".mov", ".webm", ".mkv", ".flv"] + + +def _decode_selected_frames(video: Input.Video, indices: list[int]) -> Input.Video: + """Decode only the requested frame indices from a video. + + Opens the underlying container once, decodes frames in presentation order, + keeps only the ones whose index is in ``indices``, and returns the result + wrapped in a VideoFromComponents so it still satisfies the VideoInput + contract for downstream nodes. + """ + indices_sorted = sorted(set(indices)) + max_idx = indices_sorted[-1] + source = video.get_stream_source() + + frames_by_idx: dict[int, torch.Tensor] = {} + with av.open(source, mode="r") as container: + stream = container.streams.video[0] + wanted = set(indices_sorted) + for frame_idx, frame in enumerate(container.decode(stream)): + if frame_idx in wanted: + img = frame.to_ndarray(format="rgb24") + frames_by_idx[frame_idx] = torch.from_numpy(img.copy()).float() / 255.0 + if frame_idx >= max_idx: + break + + stacked = torch.stack([frames_by_idx[i] for i in indices]) + return InputImpl.VideoFromComponents( + Types.VideoComponents(images=stacked, frame_rate=video.get_frame_rate()) + ) + + +class LoadImageDataSetFromFolderNode(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LoadImageDataSetFromFolder", + search_aliases=["load folder", "load from folder", "load dataset", "load images", "import dataset"], + display_name="Load Image (from Folder)", + category="image", + description="Load a dataset of images from a specified folder and return a list of images. Supported formats: PNG, JPG, JPEG, WEBP.", + is_experimental=True, + inputs=[ + io.Combo.Input( + "folder", + options=folder_paths.get_input_subfolders(), + tooltip="The folder to load images from.", + ) + ], + outputs=[ + io.Image.Output( + display_name="images", + is_output_list=True, + tooltip="List of loaded images", + ) + ], + ) + + @classmethod + def execute(cls, folder): + sub_input_dir = secure_subfolder_path(folder_paths.get_input_directory(), folder) + valid_extensions = [".png", ".jpg", ".jpeg", ".webp"] + image_files = [ + f + for f in os.listdir(sub_input_dir) + if any(f.lower().endswith(ext) for ext in valid_extensions) + ] + output_tensor = load_and_process_images(image_files, sub_input_dir) + return io.NodeOutput(output_tensor) + + +class LoadImageTextDataSetFromFolderNode(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LoadImageTextDataSetFromFolder", + search_aliases=["load folder", "load from folder", "load dataset", "load images", "import dataset"], + display_name="Load Image-Text (from Folder)", + category="image", + description="Load a dataset of pairs of images and text captions from a specified folder and return them as a list. Supported formats: PNG, JPG, JPEG, WEBP.", + is_experimental=True, + inputs=[ + io.Combo.Input( + "folder", + options=folder_paths.get_input_subfolders(), + tooltip="The folder to load images and text captions from.", + ) + ], + outputs=[ + io.Image.Output( + display_name="images", + is_output_list=True, + tooltip="List of loaded images", + ), + io.String.Output( + display_name="texts", + is_output_list=True, + tooltip="List of text captions", + ), + ], + ) + + @classmethod + def execute(cls, folder): + logging.info(f"Loading images from folder: {folder}") + + sub_input_dir = secure_subfolder_path(folder_paths.get_input_directory(), folder) + valid_extensions = [".png", ".jpg", ".jpeg", ".webp"] + + image_files = [] + for item in os.listdir(sub_input_dir): + path = os.path.join(sub_input_dir, item) + if any(item.lower().endswith(ext) for ext in valid_extensions): + image_files.append(path) + elif os.path.isdir(path): + # Support kohya-ss/sd-scripts folder structure + repeat = 1 + if item.split("_")[0].isdigit(): + repeat = int(item.split("_")[0]) + image_files.extend( + [ + os.path.join(path, f) + for f in os.listdir(path) + if any(f.lower().endswith(ext) for ext in valid_extensions) + ] + * repeat + ) + + caption_file_path = [ + f.replace(os.path.splitext(f)[1], ".txt") for f in image_files + ] + captions = [] + for caption_file in caption_file_path: + caption_path = os.path.join(sub_input_dir, caption_file) + if os.path.exists(caption_path): + with open(caption_path, "r", encoding="utf-8") as f: + caption = f.read().strip() + captions.append(caption) + else: + captions.append("") + + output_tensor = load_and_process_images(image_files, sub_input_dir) + + logging.info(f"Loaded {len(output_tensor)} images from {sub_input_dir}.") + return io.NodeOutput(output_tensor, captions) + + +class LoadVideoDataSetFromFolderNode(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LoadVideoDataSetFromFolder", + search_aliases=["load folder", "load from folder", "load dataset", "load videos", "import dataset"], + display_name="Load Video (from Folder)", + category="video", + description="Load a dataset of videos from a specified folder and return a list of videos. Supported formats: MP4, AVI, MOV, WEBM, MKV, FLV.", + is_experimental=True, + inputs=[ + io.Combo.Input( + "folder", + options=folder_paths.get_input_subfolders(), + tooltip="The folder containing video files.", + ), + ], + outputs=[ + io.Video.Output( + display_name="videos", + is_output_list=True, + tooltip="Lazy video references; frames are decoded only when needed downstream.", + ), + ], + ) + + @classmethod + def execute(cls, folder): + sub_input_dir = secure_subfolder_path(folder_paths.get_input_directory(), folder) + video_files = sorted([ + f for f in os.listdir(sub_input_dir) + if any(f.lower().endswith(ext) for ext in VALID_VIDEO_EXTENSIONS) + ]) + + if not video_files: + raise ValueError(f"No video files found in {sub_input_dir}") + + videos = [InputImpl.VideoFromFile(os.path.join(sub_input_dir, f)) for f in video_files] + logging.info(f"Loaded {len(videos)} lazy video references from {sub_input_dir}") + return io.NodeOutput(videos) + + +class LoadVideoTextDataSetFromFolderNode(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LoadVideoTextDataSetFromFolder", + search_aliases=["load folder", "load from folder", "load dataset", "load videos", "import dataset"], + display_name="Load Video-Text (from Folder)", + category="video", + description="Load a dataset of pairs of videos and text captions from a specified folder and return them as a list. Supported formats: MP4, AVI, MOV, WEBM, MKV, FLV.", + is_experimental=True, + inputs=[ + io.Combo.Input( + "folder", + options=folder_paths.get_input_subfolders(), + tooltip="The folder containing video files and .txt captions.", + ), + ], + outputs=[ + io.Video.Output( + display_name="videos", + is_output_list=True, + tooltip="Lazy video references; frames are decoded only when needed downstream.", + ), + io.String.Output( + display_name="texts", + is_output_list=True, + tooltip="List of text captions.", + ), + ], + ) + + @classmethod + def execute(cls, folder): + sub_input_dir = secure_subfolder_path(folder_paths.get_input_directory(), folder) + + video_files = [] + for item in sorted(os.listdir(sub_input_dir)): + path = os.path.join(sub_input_dir, item) + if any(item.lower().endswith(ext) for ext in VALID_VIDEO_EXTENSIONS): + video_files.append(path) + elif os.path.isdir(path): + # Support kohya-ss/sd-scripts folder structure: {repeat}_{desc}/ + repeat = 1 + if item.split("_")[0].isdigit(): + repeat = int(item.split("_")[0]) + video_files.extend([ + os.path.join(path, f) + for f in sorted(os.listdir(path)) + if any(f.lower().endswith(ext) for ext in VALID_VIDEO_EXTENSIONS) + ] * repeat) + + if not video_files: + raise ValueError(f"No video files found in {sub_input_dir}") + + captions = [] + for vf in video_files: + caption_path = os.path.splitext(vf)[0] + ".txt" + if os.path.exists(caption_path): + with open(caption_path, "r", encoding="utf-8") as f: + captions.append(f.read().strip()) + else: + captions.append("") + + videos = [InputImpl.VideoFromFile(vf) for vf in video_files] + logging.info(f"Loaded {len(videos)} lazy video references with captions from {sub_input_dir}") + return io.NodeOutput(videos, captions) + + +def save_images_to_folder(image_list, output_dir, prefix="image", overwrite=True): + """Utility function to save a list of image tensors to disk. + + Args: + image_list: List of image tensors (each [1, H, W, C] or [H, W, C] or [C, H, W]) + output_dir: Directory to save images to + prefix: Filename prefix + + Returns: + List of saved filenames + """ + os.makedirs(output_dir, exist_ok=True) + saved_files = [] + + for idx, img_tensor in enumerate(image_list): + # Handle different tensor shapes + if isinstance(img_tensor, torch.Tensor): + # Remove batch dimension if present [1, H, W, C] -> [H, W, C] + if img_tensor.dim() == 4 and img_tensor.shape[0] == 1: + img_tensor = img_tensor.squeeze(0) + + # If tensor is [C, H, W], permute to [H, W, C] + if img_tensor.dim() == 3 and img_tensor.shape[0] in [1, 3, 4]: + if ( + img_tensor.shape[0] <= 4 + and img_tensor.shape[1] > 4 + and img_tensor.shape[2] > 4 + ): + img_tensor = img_tensor.permute(1, 2, 0) + + # Convert to numpy and scale to 0-255 + img_array = img_tensor.cpu().numpy() + img_array = np.clip(img_array * 255.0, 0, 255).astype(np.uint8) + + # Convert to PIL Image + img = Image.fromarray(img_array) + else: + raise ValueError(f"Expected torch.Tensor, got {type(img_tensor)}") + + # Save image + if overwrite: + filename = f"{prefix}_{idx:05d}.png" + else: + _, _, counter, _, resolved_prefix = folder_paths.get_save_image_path(prefix, output_dir) + filename = f"{resolved_prefix}_{counter:05}_{idx:05d}.png" + filepath = os.path.join(output_dir, filename) + img.save(filepath) + saved_files.append(filename) + + return saved_files + + +class SaveImageDataSetToFolderNode(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SaveImageDataSetToFolder", + search_aliases=["save folder", "save to folder", "save dataset", "save images", "export dataset"], + display_name="Save Image (to Folder) (DEPRECATED)", + category="image", + description="Save a dataset of images to a specified folder. Supported formats: PNG.", + is_experimental=True, + is_output_node=True, + is_input_list=True, # Receive images as list + inputs=[ + io.Image.Input("images", tooltip="List of images to save."), + io.String.Input( + "folder_name", + default="dataset", + tooltip="Name of the folder to save images to (inside output directory).", + ), + io.String.Input( + "filename_prefix", + default="image", + tooltip="Prefix for saved image filenames.", + advanced=True, + ), + io.Combo.Input( + "mode", + default="overwrite", + options=["overwrite", "increment"], + tooltip="Whether to overwrite existing files or increment filenames to avoid overwriting." + ), + ], + outputs=[], + is_deprecated=True, # This node is redundant and superseded by existing Save Image nodes where the target folder can be specified in the filename_prefix + ) + + @classmethod + def execute(cls, images, folder_name, filename_prefix, mode): + # Extract scalar values + folder_name = folder_name[0] + filename_prefix = filename_prefix[0] + mode = mode[0] + + output_dir = secure_subfolder_path(folder_paths.get_output_directory(), folder_name) + saved_files = save_images_to_folder(images, output_dir, filename_prefix, mode=='overwrite') + + logging.info(f"Saved {len(saved_files)} images to {output_dir}.") + return io.NodeOutput() + + +class SaveImageTextDataSetToFolderNode(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SaveImageTextDataSetToFolder", + search_aliases=["save folder", "save to folder", "save dataset", "save images", "save text", "export dataset"], + display_name="Save Image-Text (to Folder)", + category="image", + description="Save a dataset of pairs of images and text captions to a specified folder. Images are saved as PNG files and captions are saved as TXT files with the same filename_prefix.", + is_experimental=True, + is_output_node=True, + is_input_list=True, # Receive both images and texts as lists + inputs=[ + io.Image.Input("images", tooltip="List of images to save."), + io.String.Input("texts", + optional=True, + force_input=True, + tooltip="List of text captions to save." + ), + io.String.Input( + "folder_name", + default="dataset", + tooltip="Name of the folder to save images to (inside output directory).", + ), + io.String.Input( + "filename_prefix", + default="image", + tooltip="Prefix for saved image filenames.", + advanced=True, + ), + io.Combo.Input( + "mode", + default="overwrite", + options=["overwrite", "increment"], + tooltip="Whether to overwrite existing files or increment filenames to avoid overwriting." + ), + ], + outputs=[], + ) + + @classmethod + def execute(cls, images, folder_name, filename_prefix, mode, texts=None): + # Extract scalar values + folder_name = folder_name[0] + filename_prefix = filename_prefix[0] + mode = mode[0] + + output_dir = secure_subfolder_path(folder_paths.get_output_directory(), folder_name) + saved_files = save_images_to_folder(images, output_dir, filename_prefix, mode=='overwrite') + + # Save captions + if texts: + for idx, (filename, caption) in enumerate(zip(saved_files, texts)): + caption_filename = filename.replace(".png", ".txt") + caption_path = os.path.join(output_dir, caption_filename) + with open(caption_path, "w", encoding="utf-8") as f: + f.write(caption) + + logging.info(f"Saved {len(saved_files)} images and captions to {output_dir}.") + return io.NodeOutput() + + +# ========== Helper Functions for Transform Nodes ========== + + +def tensor_to_pil(img_tensor): + """Convert tensor to PIL Image.""" + if img_tensor.dim() == 4 and img_tensor.shape[0] == 1: + img_tensor = img_tensor.squeeze(0) + img_array = (img_tensor.cpu().numpy() * 255).clip(0, 255).astype(np.uint8) + return Image.fromarray(img_array) + + +def pil_to_tensor(img): + """Convert PIL Image to tensor.""" + img_array = np.array(img).astype(np.float32) / 255.0 + return torch.from_numpy(img_array)[None,] + + +# ========== Base Classes for Transform Nodes ========== + + +class ImageProcessingNode(io.ComfyNode): + """Base class for image processing nodes that operate on images. + + Child classes should set: + node_id: Unique node identifier (required) + search_aliases: List of search aliases (optional) + display_name: Display name (optional, defaults to node_id) + description: Node description (optional) + extra_inputs: List of additional io.Input objects beyond "images" (optional) + is_group_process: None (auto-detect), True (group), or False (individual) (optional) + is_output_list: True (list output) or False (single output) (optional, default True) + is_deprecated: True if the node is deprecated (optional, default False) + + Child classes must implement ONE of: + _process(cls, image, **kwargs) -> tensor (for single-item processing) + _group_process(cls, images, **kwargs) -> list[tensor] (for group processing) + """ + + node_id = None + search_aliases = [] + display_name = None + description = None + extra_inputs = [] + is_group_process = None # None = auto-detect, True/False = explicit + is_output_list = None # None = auto-detect based on processing mode + is_deprecated = False + @classmethod + def _detect_processing_mode(cls): + """Detect whether this node uses group or individual processing. + + Returns: + bool: True if group processing, False if individual processing + """ + # Explicit setting takes precedence + if cls.is_group_process is not None: + return cls.is_group_process + + # Check which method is overridden by looking at the defining class in MRO + base_class = ImageProcessingNode + + # Find which class in MRO defines _process + process_definer = None + for klass in cls.__mro__: + if "_process" in klass.__dict__: + process_definer = klass + break + + # Find which class in MRO defines _group_process + group_definer = None + for klass in cls.__mro__: + if "_group_process" in klass.__dict__: + group_definer = klass + break + + # Check what was overridden (not defined in base class) + has_process = process_definer is not None and process_definer is not base_class + has_group = group_definer is not None and group_definer is not base_class + + if has_process and has_group: + raise ValueError( + f"{cls.__name__}: Cannot override both _process and _group_process. " + "Override only one, or set is_group_process explicitly." + ) + if not has_process and not has_group: + raise ValueError( + f"{cls.__name__}: Must override either _process or _group_process" + ) + + return has_group + + @classmethod + def _ensure_image_list(cls, images): + """Normalize to a flat list of [1, H, W, C] tensors.""" + if isinstance(images, torch.Tensor): + if images.ndim != 4: + raise ValueError(f"Expected 4D image tensor, got shape {tuple(images.shape)}") + return [images[i:i+1] for i in range(images.shape[0])] + + flat = [] + for item in images: + if not isinstance(item, torch.Tensor) or item.ndim != 4: + raise ValueError(f"Expected 4D image tensor, got {type(item).__name__} shape {getattr(item, 'shape', None)}") + flat.extend([item[i:i+1] for i in range(item.shape[0])]) + return flat + + @classmethod + def define_schema(cls): + if cls.node_id is None: + raise NotImplementedError(f"{cls.__name__} must set node_id class variable") + + is_group = cls._detect_processing_mode() + + # Auto-detect is_output_list if not explicitly set + # Single processing: False (backend collects results into list) + # Group processing: True by default (can be False for single-output nodes) + output_is_list = ( + cls.is_output_list if cls.is_output_list is not None else is_group + ) + + inputs = [ + io.Image.Input( + "images", + tooltip=( + "List of images to process." if is_group else "Image to process." + ), + ) + ] + inputs.extend(cls.extra_inputs) + + return io.Schema( + node_id=cls.node_id, + search_aliases=cls.search_aliases, + display_name=cls.display_name or cls.node_id, + category=cls.category, + description=cls.description, + is_experimental=True, + is_input_list=is_group, # True for group, False for individual + inputs=inputs, + outputs=[ + io.Image.Output( + display_name="images", + is_output_list=output_is_list, + tooltip="Processed images", + ) + ], + ) + + @classmethod + def execute(cls, images, **kwargs): + """Execute the node. Routes to _process or _group_process based on mode. + + For individual processing (_process), automatically handles multi-frame + inputs (video tensors [T, H, W, C]) by applying _process per-frame and + concatenating the results. This allows all spatial transform nodes to + work with video without modification. Nodes that natively handle batched + tensors (e.g. pure tensor math) can set per_frame_process = False to + skip the per-frame loop. + """ + is_group = cls._detect_processing_mode() + + if is_group: + images = cls._ensure_image_list(images) + + # Extract scalar values from lists for parameters + params = {} + for k, v in kwargs.items(): + if isinstance(v, list) and len(v) == 1: + params[k] = v[0] + else: + params[k] = v + + if is_group: + # Group processing: images is list, call _group_process + result = cls._group_process(images, **params) + else: + # Individual processing: images is single item, call _process + # Auto-loop over frames for multi-frame inputs (video [T, H, W, C]) + # so that PIL-based spatial transforms work per-frame automatically. + if images.shape[0] > 1 and getattr(cls, 'per_frame_process', True): + results = [] + for i in range(images.shape[0]): + frame_result = cls._process(images[i:i + 1], **params) + results.append(frame_result) + result = torch.cat(results, dim=0) + else: + result = cls._process(images, **params) + + return io.NodeOutput(result) + + @classmethod + def _process(cls, image, **kwargs): + """Override this method for single-item processing. + + Args: + image: tensor - Single image tensor + **kwargs: Additional parameters (already extracted from lists) + + Returns: + tensor - Processed image + """ + raise NotImplementedError(f"{cls.__name__} must implement _process method") + + @classmethod + def _group_process(cls, images, **kwargs): + """Override this method for group processing. + + Args: + images: list[tensor] - List of image tensors + **kwargs: Additional parameters (already extracted from lists) + + Returns: + list[tensor] - Processed images + """ + raise NotImplementedError( + f"{cls.__name__} must implement _group_process method" + ) + + +class TextProcessingNode(io.ComfyNode): + """Base class for text processing nodes that operate on texts. + + Child classes should set: + node_id: Unique node identifier (required) + search_aliases: List of search aliases (optional) + display_name: Display name (optional, defaults to node_id) + description: Node description (optional) + extra_inputs: List of additional io.Input objects beyond "texts" (optional) + is_group_process: None (auto-detect), True (group), or False (individual) (optional) + is_output_list: True (list output) or False (single output) (optional, default True) + is_deprecated: True if the node is deprecated (optional, default False) + + Child classes must implement ONE of: + _process(cls, text, **kwargs) -> str (for single-item processing) + _group_process(cls, texts, **kwargs) -> list[str] (for group processing) + """ + + node_id = None + search_aliases = [] + display_name = None + description = None + extra_inputs = [] + is_group_process = None # None = auto-detect, True/False = explicit + is_output_list = None # None = auto-detect based on processing mode + is_deprecated = False + @classmethod + def _detect_processing_mode(cls): + """Detect whether this node uses group or individual processing. + + Returns: + bool: True if group processing, False if individual processing + """ + # Explicit setting takes precedence + if cls.is_group_process is not None: + return cls.is_group_process + + # Check which method is overridden by looking at the defining class in MRO + base_class = TextProcessingNode + + # Find which class in MRO defines _process + process_definer = None + for klass in cls.__mro__: + if "_process" in klass.__dict__: + process_definer = klass + break + + # Find which class in MRO defines _group_process + group_definer = None + for klass in cls.__mro__: + if "_group_process" in klass.__dict__: + group_definer = klass + break + + # Check what was overridden (not defined in base class) + has_process = process_definer is not None and process_definer is not base_class + has_group = group_definer is not None and group_definer is not base_class + + if has_process and has_group: + raise ValueError( + f"{cls.__name__}: Cannot override both _process and _group_process. " + "Override only one, or set is_group_process explicitly." + ) + if not has_process and not has_group: + raise ValueError( + f"{cls.__name__}: Must override either _process or _group_process" + ) + + return has_group + + @classmethod + def define_schema(cls): + if cls.node_id is None: + raise NotImplementedError(f"{cls.__name__} must set node_id class variable") + + is_group = cls._detect_processing_mode() + + inputs = [ + io.String.Input( + "texts", + tooltip="List of texts to process." if is_group else "Text to process.", + ) + ] + inputs.extend(cls.extra_inputs) + + return io.Schema( + node_id=cls.node_id, + display_name=cls.display_name or cls.node_id, + category="text", + is_experimental=True, + is_input_list=is_group, # True for group, False for individual + inputs=inputs, + outputs=[ + io.String.Output( + display_name="texts", + is_output_list=cls.is_output_list, + tooltip="Processed texts", + ) + ], + ) + + @classmethod + def execute(cls, texts, **kwargs): + """Execute the node. Routes to _process or _group_process based on mode.""" + is_group = cls._detect_processing_mode() + + # Extract scalar values from lists for parameters + params = {} + for k, v in kwargs.items(): + if isinstance(v, list) and len(v) == 1: + params[k] = v[0] + else: + params[k] = v + + if is_group: + # Group processing: texts is list, call _group_process + result = cls._group_process(texts, **params) + else: + # Individual processing: texts is single item, call _process + result = cls._process(texts, **params) + + # Wrap result based on is_output_list + if cls.is_output_list: + # Result should already be a list (or will be for individual) + return io.NodeOutput(result if is_group else [result]) + else: + # Single output - wrap in list for NodeOutput + return io.NodeOutput([result]) + + @classmethod + def _process(cls, text, **kwargs): + """Override this method for single-item processing. + + Args: + text: str - Single text string + **kwargs: Additional parameters (already extracted from lists) + + Returns: + str - Processed text + """ + raise NotImplementedError(f"{cls.__name__} must implement _process method") + + @classmethod + def _group_process(cls, texts, **kwargs): + """Override this method for group processing. + + Args: + texts: list[str] - List of text strings + **kwargs: Additional parameters (already extracted from lists) + + Returns: + list[str] - Processed texts + """ + raise NotImplementedError( + f"{cls.__name__} must implement _group_process method" + ) + + +# ========== Image Transform Nodes ========== + + +class ResizeImagesByShorterEdgeNode(ImageProcessingNode): + node_id = "ResizeImagesByShorterEdge" + display_name = "Resize Images by Shorter Edge (DEPRECATED)" + category = "image/transform" + description = "Resize images so that the shorter edge matches the specified dimension while preserving aspect ratio." + is_deprecated = True # This node is superseded by Resize Image/Mask with resize_type = scale shorter dimension + extra_inputs = [ + io.Int.Input( + "shorter_edge", + default=512, + min=1, + max=8192, + tooltip="Target dimension for the shorter edge.", + ), + ] + + @classmethod + def _process(cls, image, shorter_edge): + img = tensor_to_pil(image) + w, h = img.size + if w < h: + new_w = shorter_edge + new_h = int(h * (shorter_edge / w)) + else: + new_h = shorter_edge + new_w = int(w * (shorter_edge / h)) + img = img.resize((new_w, new_h), Image.Resampling.LANCZOS) + return pil_to_tensor(img) + + +class ResizeImagesByLongerEdgeNode(ImageProcessingNode): + node_id = "ResizeImagesByLongerEdge" + display_name = "Resize Images by Longer Edge (DEPRECATED)" + category = "image/transform" + description = "Resize images so that the longer edge matches the specified dimension while preserving aspect ratio." + is_deprecated = True # This node is superseded by Resize Image/Mask with resize_type = scale longer dimension + extra_inputs = [ + io.Int.Input( + "longer_edge", + default=1024, + min=1, + max=8192, + tooltip="Target dimension for the longer edge.", + ), + ] + + @classmethod + def _process(cls, image, longer_edge): + resized_images = [] + for image_i in image: + img = tensor_to_pil(image_i) + w, h = img.size + if w > h: + new_w = longer_edge + new_h = int(h * (longer_edge / w)) + else: + new_h = longer_edge + new_w = int(w * (longer_edge / h)) + img = img.resize((new_w, new_h), Image.Resampling.LANCZOS) + resized_images.append(pil_to_tensor(img)) + return torch.cat(resized_images, dim=0) + + +class CenterCropImagesNode(ImageProcessingNode): + node_id = "CenterCropImages" + search_aliases=["crop", "cut", "trim"] + display_name="Crop Image (Center)" + category="image/transform" + description = "Center crop an image to the specified dimensions." + extra_inputs = [ + io.Int.Input("width", default=512, min=1, max=8192, tooltip="Crop width."), + io.Int.Input("height", default=512, min=1, max=8192, tooltip="Crop height."), + ] + + @classmethod + def _process(cls, image, width, height): + img = tensor_to_pil(image) + left = max(0, (img.width - width) // 2) + top = max(0, (img.height - height) // 2) + right = min(img.width, left + width) + bottom = min(img.height, top + height) + img = img.crop((left, top, right, bottom)) + return pil_to_tensor(img) + + +class RandomCropImagesNode(ImageProcessingNode): + node_id = "RandomCropImages" + search_aliases=["crop", "cut", "trim"] + display_name = "Crop Image (Random)" + category="image/transform" + description = "Randomly crop an image to the specified dimensions." + + extra_inputs = [ + io.Int.Input("width", default=512, min=1, max=8192, tooltip="Crop width."), + io.Int.Input("height", default=512, min=1, max=8192, tooltip="Crop height."), + io.Int.Input( + "seed", default=0, min=0, max=0xFFFFFFFFFFFFFFFF, tooltip="Random seed." + ), + ] + + @classmethod + def _process(cls, image, width, height, seed): + np.random.seed(seed % (2**32 - 1)) + img = tensor_to_pil(image) + max_left = max(0, img.width - width) + max_top = max(0, img.height - height) + left = np.random.randint(0, max_left + 1) if max_left > 0 else 0 + top = np.random.randint(0, max_top + 1) if max_top > 0 else 0 + right = min(img.width, left + width) + bottom = min(img.height, top + height) + img = img.crop((left, top, right, bottom)) + return pil_to_tensor(img) + + +class NormalizeImagesNode(ImageProcessingNode): + node_id = "NormalizeImages" + search_aliases=["normalize", "normalize colors"] + display_name = "Normalize Image Colors" + category = "image/color" + description = "Normalize images using mean and standard deviation." + per_frame_process = False # Pure tensor math, handles any batch size + extra_inputs = [ + io.Float.Input( + "mean", + default=0.5, + min=0.0, + max=1.0, + tooltip="Mean value for normalization.", + advanced=True, + ), + io.Float.Input( + "std", + default=0.5, + min=0.001, + max=1.0, + tooltip="Standard deviation for normalization.", + advanced=True, + ), + ] + + @classmethod + def _process(cls, image, mean, std): + return (image - mean) / std + + +class AdjustBrightnessNode(ImageProcessingNode): + node_id = "AdjustBrightness" + search_aliases=["brightness"] + display_name = "Adjust Brightness" + category="image/adjustments" + description = "Adjust the brightness of an image." + per_frame_process = False # Pure tensor math, handles any batch size + extra_inputs = [ + io.Float.Input( + "factor", + default=1.0, + min=0.0, + max=2.0, + tooltip="Brightness factor. 1.0 = no change, <1.0 = darker, >1.0 = brighter.", + ), + ] + + @classmethod + def _process(cls, image, factor): + return (image * factor).clamp(0.0, 1.0) + + +class AdjustContrastNode(ImageProcessingNode): + node_id = "AdjustContrast" + search_aliases=["contrast"] + display_name = "Adjust Contrast" + category="image/adjustments" + description = "Adjust the contrast of an image." + per_frame_process = False # Pure tensor math, handles any batch size + extra_inputs = [ + io.Float.Input( + "factor", + default=1.0, + min=0.0, + max=2.0, + tooltip="Contrast factor. 1.0 = no change, <1.0 = less contrast, >1.0 = more contrast.", + ), + ] + + @classmethod + def _process(cls, image, factor): + return ((image - 0.5) * factor + 0.5).clamp(0.0, 1.0) + + +class ShuffleDatasetNode(ImageProcessingNode): + node_id = "ShuffleDataset" + search_aliases=["shuffle", "randomize", "mix"] + display_name = "Shuffle Images List" + category = "image/batch" + description = "Randomly shuffle the order of images in a list." + is_group_process = True # Requires full list to shuffle + extra_inputs = [ + io.Int.Input( + "seed", default=0, min=0, max=0xFFFFFFFFFFFFFFFF, tooltip="Random seed." + ), + ] + + @classmethod + def _group_process(cls, images, seed): + np.random.seed(seed % (2**32 - 1)) + indices = np.random.permutation(len(images)) + return [images[i] for i in indices] + + +class ShuffleImageTextDatasetNode(io.ComfyNode): + """Special node that shuffles both images and texts together.""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ShuffleImageTextDataset", + search_aliases=["shuffle", "randomize", "mix"], + display_name = "Shuffle Pairs of Image-Text", + category = "image/batch", + description = "Randomly shuffle the order of pairs of image-text in a list.", + is_experimental=True, + is_input_list=True, + inputs=[ + io.Image.Input("images", tooltip="List of images to shuffle."), + io.String.Input("texts", tooltip="List of texts to shuffle.", force_input=True), + io.Int.Input( + "seed", + default=0, + min=0, + max=0xFFFFFFFFFFFFFFFF, + tooltip="Random seed.", + ), + ], + outputs=[ + io.Image.Output( + display_name="images", + is_output_list=True, + tooltip="Shuffled images", + ), + io.String.Output( + display_name="texts", is_output_list=True, tooltip="Shuffled texts" + ), + ], + ) + + @classmethod + def execute(cls, images, texts, seed): + seed = seed[0] # Extract scalar + np.random.seed(seed % (2**32 - 1)) + indices = np.random.permutation(len(images)) + shuffled_images = [images[i] for i in indices] + shuffled_texts = [texts[i] for i in indices] + return io.NodeOutput(shuffled_images, shuffled_texts) + + +# ========== Video Processing Nodes ========== + + +class VideoFrameSampleNode(io.ComfyNode): + """Sample a fixed number of frames from a video using various strategies. + + For contiguous strategies ("head"/"tail") the result is a fully lazy + VideoInput (no frames decoded). For non-contiguous strategies + ("uniform"/"random") only the selected indices are decoded. + """ + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="VideoFrameSample", + search_aliases=["sample frames", "extract frames"], + display_name="Sample Video Frame", + category="video", + description="Sample a fixed number of frames from a video using various strategies.", + is_experimental=True, + inputs=[ + io.Video.Input("video", tooltip="Input video."), + io.Int.Input( + "num_frames", + default=16, + min=1, + max=9999, + tooltip="Number of frames to sample.", + ), + io.Combo.Input( + "strategy", + options=["uniform", "head", "tail", "random"], + default="uniform", + tooltip="uniform: evenly spaced, head: first N, tail: last N, random: random sorted.", + ), + io.Int.Input( + "seed", + default=0, + min=0, + max=0xFFFFFFFFFFFFFFFF, + tooltip="Random seed (only used with 'random' strategy).", + ), + ], + outputs=[ + io.Video.Output(display_name="video", tooltip="Sampled video."), + ], + ) + + @classmethod + def execute(cls, video, num_frames, strategy, seed): + total_frames = video.get_frame_count() + num_frames = min(num_frames, total_frames) + fps = float(video.get_frame_rate()) + + if strategy == "head": + return io.NodeOutput( + video.as_trimmed(0.0, num_frames / fps, strict_duration=False) + ) + if strategy == "tail": + start_t = (total_frames - num_frames) / fps + return io.NodeOutput( + video.as_trimmed(start_t, num_frames / fps, strict_duration=False) + ) + + if strategy == "uniform": + if num_frames == 1: + indices = [total_frames // 2] + else: + indices = [round(i * (total_frames - 1) / (num_frames - 1)) for i in range(num_frames)] + elif strategy == "random": + rng = np.random.RandomState(seed % (2**32 - 1)) + indices = sorted(rng.choice(total_frames, size=num_frames, replace=False).tolist()) + else: + raise ValueError(f"Unknown strategy: {strategy}") + + return io.NodeOutput(_decode_selected_frames(video, indices)) + + +class VideoTemporalCropNode(io.ComfyNode): + """Crop a continuous range of frames from a video (fully lazy).""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="VideoTemporalCrop", + search_aliases=["crop", "crop video", "temporal crop", "truncate video"], + display_name="Crop Video (Temporal)", + category="video/transform", + description="Crop a continuous range of frames from a video.", + is_experimental=True, + inputs=[ + io.Video.Input("video", tooltip="Input video."), + io.Int.Input( + "start_frame", + default=0, + min=0, + max=99999, + tooltip="Starting frame index.", + ), + io.Int.Input( + "length", + default=16, + min=1, + max=99999, + tooltip="Number of frames to keep.", + ), + ], + outputs=[ + io.Video.Output(display_name="video", tooltip="Cropped video (lazy)."), + ], + ) + + @classmethod + def execute(cls, video, start_frame, length): + total_frames = video.get_frame_count() + fps = float(video.get_frame_rate()) + start_frame = min(start_frame, max(total_frames - 1, 0)) + length = min(length, total_frames - start_frame) + return io.NodeOutput( + video.as_trimmed(start_frame / fps, length / fps, strict_duration=False) + ) + + +class VideoRandomTemporalCropNode(io.ComfyNode): + """Randomly crop a continuous range of frames from a video (fully lazy).""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="VideoRandomTemporalCrop", + search_aliases=["crop", "crop video", "temporal crop", "truncate video", "random crop"], + display_name="Crop Video (Temporal Random)", + category="video/transform", + description="Randomly crop a continuous range of frames from a video.", + is_experimental=True, + inputs=[ + io.Video.Input("video", tooltip="Input video."), + io.Int.Input( + "length", + default=16, + min=1, + max=99999, + tooltip="Number of frames to keep.", + ), + io.Int.Input( + "seed", + default=0, + min=0, + max=0xFFFFFFFFFFFFFFFF, + tooltip="Random seed.", + ), + ], + outputs=[ + io.Video.Output(display_name="video", tooltip="Cropped video (lazy)."), + ], + ) + + @classmethod + def execute(cls, video, length, seed): + total_frames = video.get_frame_count() + fps = float(video.get_frame_rate()) + length = min(length, total_frames) + max_start = total_frames - length + rng = np.random.RandomState(seed % (2**32 - 1)) + start = rng.randint(0, max_start + 1) if max_start > 0 else 0 + return io.NodeOutput( + video.as_trimmed(start / fps, length / fps, strict_duration=False) + ) + + +class ShuffleVideoDatasetNode(io.ComfyNode): + """Randomly shuffle the order of videos in the dataset.""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ShuffleVideoDataset", + search_aliases=["shuffle", "randomize", "mix"], + display_name="Shuffle Videos List", + category="video/batch", + description="Randomly shuffle the order of videos in a list.", + is_experimental=True, + is_input_list=True, + inputs=[ + io.Video.Input("videos", tooltip="List of videos to shuffle."), + io.Int.Input( + "seed", default=0, min=0, max=0xFFFFFFFFFFFFFFFF, tooltip="Random seed." + ), + ], + outputs=[ + io.Video.Output( + display_name="videos", + is_output_list=True, + tooltip="Shuffled videos", + ), + ], + ) + + @classmethod + def execute(cls, videos, seed): + seed = seed[0] if isinstance(seed, list) else seed + np.random.seed(seed % (2**32 - 1)) + indices = np.random.permutation(len(videos)) + return io.NodeOutput([videos[i] for i in indices]) + + +class ShuffleVideoTextDatasetNode(io.ComfyNode): + """Shuffle videos and their captions together, preserving pairs.""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ShuffleVideoTextDataset", + search_aliases=["shuffle", "randomize", "mix"], + display_name="Shuffle Pairs of Video-Text", + category="dataset/video", + description="Randomly shuffle the order of pairs of video-text in a list.", + is_experimental=True, + is_input_list=True, + inputs=[ + io.Video.Input("videos", tooltip="List of videos to shuffle."), + io.String.Input("texts", tooltip="List of texts to shuffle."), + io.Int.Input( + "seed", + default=0, + min=0, + max=0xFFFFFFFFFFFFFFFF, + tooltip="Random seed.", + ), + ], + outputs=[ + io.Video.Output( + display_name="videos", + is_output_list=True, + tooltip="Shuffled videos", + ), + io.String.Output( + display_name="texts", + is_output_list=True, + tooltip="Shuffled texts", + ), + ], + ) + + @classmethod + def execute(cls, videos, texts, seed): + seed = seed[0] if isinstance(seed, list) else seed + np.random.seed(seed % (2**32 - 1)) + indices = np.random.permutation(len(videos)) + return io.NodeOutput( + [videos[i] for i in indices], + [texts[i] for i in indices], + ) + + +# ========== Text Transform Nodes ========== + + +class TextToLowercaseNode(TextProcessingNode): + node_id = "TextToLowercase" + search_aliases=["lowercase"] + display_name = "Convert Text to Lowercase (DEPRECATED)" + category = "text" + description = "Convert text to lowercase." + is_deprecated = True # This node is superseded by the Convert Text Case node + + @classmethod + def _process(cls, text): + return text.lower() + + +class TextToUppercaseNode(TextProcessingNode): + node_id = "TextToUppercase" + search_aliases=["uppercase"] + display_name = "Convert Text to Uppercase (DEPRECATED)" + category = "text" + description = "Convert text to uppercase." + is_deprecated = True # This node is superseded by the Convert Text Case node + + @classmethod + def _process(cls, text): + return text.upper() + + +class TruncateTextNode(TextProcessingNode): + node_id = "TruncateText" + search_aliases=["truncate", "cut", "shorten"] + display_name = "Truncate Text" + category = "text" + description = "Truncate text to a maximum length." + extra_inputs = [ + io.Int.Input( + "max_length", default=77, min=1, max=10000, tooltip="Maximum text length." + ), + ] + + @classmethod + def _process(cls, text, max_length): + return text[:max_length] + + +class AddTextPrefixNode(TextProcessingNode): + node_id = "AddTextPrefix" + display_name = "Add Text Prefix (DEPRECATED)" + category = "text" + description = "Add a prefix to all texts." + is_deprecated = True # This node is superseded by the Concatenate Text node + extra_inputs = [ + io.String.Input("prefix", default="", tooltip="Prefix to add."), + ] + + @classmethod + def _process(cls, text, prefix): + return prefix + text + + +class AddTextSuffixNode(TextProcessingNode): + node_id = "AddTextSuffix" + display_name = "Add Text Suffix (DEPRECATED)" + category = "text" + description = "Add a suffix to all texts." + is_deprecated = True # This node is superseded by the Concatenate Text node + extra_inputs = [ + io.String.Input("suffix", default="", tooltip="Suffix to add."), + ] + + @classmethod + def _process(cls, text, suffix): + return text + suffix + + +class ReplaceTextNode(TextProcessingNode): + node_id = "ReplaceText" + display_name = "Replace Text (DEPRECATED)" + category = "text" + description = "Replace text in all texts." + is_deprecated = True # This node is superseded by the other Replace Text node + extra_inputs = [ + io.String.Input("find", default="", tooltip="Text to find."), + io.String.Input("replace", default="", tooltip="Text to replace with."), + ] + + @classmethod + def _process(cls, text, find, replace): + return text.replace(find, replace) + + +class StripWhitespaceNode(TextProcessingNode): + node_id = "StripWhitespace" + display_name = "Strip Whitespace (DEPRECATED)" + category = "text" + description = "Strip leading and trailing whitespace from all texts." + is_deprecated = True # This node is superseded by the Trim Text node + + @classmethod + def _process(cls, text): + return text.strip() + + +# ========== Group Processing Example Nodes ========== + + +class ImageDeduplicationNode(ImageProcessingNode): + """Remove duplicate or very similar images from a list using perceptual hashing.""" + + node_id = "ImageDeduplication" + search_aliases=["deduplicate", "remove duplicates", "similarity filter"] + display_name = "Deduplicate Images" + category = "image/batch" + description = "Remove duplicate or very similar images from a list." + is_group_process = True # Requires full list to compare images + extra_inputs = [ + io.Float.Input( + "similarity_threshold", + default=0.95, + min=0.0, + max=1.0, + tooltip="Similarity threshold (0-1). Higher means more similar. Images above this threshold are considered duplicates.", + advanced=True, + ), + ] + + @classmethod + def _group_process(cls, images, similarity_threshold): + """Remove duplicate images using perceptual hashing.""" + if len(images) == 0: + return [] + + # Compute simple perceptual hash for each image + def compute_hash(img_tensor): + """Compute a simple perceptual hash by resizing to 8x8 and comparing to average.""" + img = tensor_to_pil(img_tensor) + # Resize to 8x8 + img_small = img.resize((8, 8), Image.Resampling.LANCZOS).convert("L") + # Get pixels + pixels = list(img_small.getdata()) + # Compute average + avg = sum(pixels) / len(pixels) + # Create hash (1 if above average, 0 otherwise) + hash_bits = "".join("1" if p > avg else "0" for p in pixels) + return hash_bits + + def hamming_distance(hash1, hash2): + """Compute Hamming distance between two hash strings.""" + return sum(c1 != c2 for c1, c2 in zip(hash1, hash2)) + + # Compute hashes for all images + hashes = [compute_hash(img) for img in images] + + # Find duplicates + keep_indices = [] + for i in range(len(images)): + is_duplicate = False + for j in keep_indices: + # Compare hashes + distance = hamming_distance(hashes[i], hashes[j]) + similarity = 1.0 - (distance / 64.0) # 64 bits total + if similarity >= similarity_threshold: + is_duplicate = True + logging.info( + f"Image {i} is similar to image {j} (similarity: {similarity:.3f}), skipping" + ) + break + + if not is_duplicate: + keep_indices.append(i) + + # Return only unique images + unique_images = [images[i] for i in keep_indices] + logging.info( + f"Deduplication: kept {len(unique_images)} out of {len(images)} images" + ) + return unique_images + + +class ImageGridNode(ImageProcessingNode): + """Combine multiple images into a single grid/collage.""" + + node_id = "ImageGrid" + search_aliases=["grid", "collage", "combine"] + display_name = "Make Image Grid" + category="image/batch" + description = "Arrange multiple images into a grid layout." + is_group_process = True # Requires full list to create grid + is_output_list = False # Outputs single grid image + extra_inputs = [ + io.Int.Input( + "columns", + default=4, + min=1, + max=20, + tooltip="Number of columns in the grid.", + ), + io.Int.Input( + "cell_width", + default=256, + min=32, + max=2048, + tooltip="Width of each cell in the grid.", + advanced=True, + ), + io.Int.Input( + "cell_height", + default=256, + min=32, + max=2048, + tooltip="Height of each cell in the grid.", + advanced=True, + ), + io.Int.Input( + "padding", default=4, min=0, max=50, tooltip="Padding between images.", advanced=True + ), + ] + + @classmethod + def _group_process(cls, images, columns, cell_width, cell_height, padding): + """Arrange images into a grid.""" + if len(images) == 0: + raise ValueError("Cannot create grid from empty image list") + + # Calculate grid dimensions + num_images = len(images) + rows = (num_images + columns - 1) // columns # Ceiling division + + # Calculate total grid size + grid_width = columns * cell_width + (columns - 1) * padding + grid_height = rows * cell_height + (rows - 1) * padding + + # Create blank grid + grid = Image.new("RGB", (grid_width, grid_height), (0, 0, 0)) + + # Place images + for idx, img_tensor in enumerate(images): + row = idx // columns + col = idx % columns + + # Convert to PIL and resize to cell size + img = tensor_to_pil(img_tensor) + img = img.resize((cell_width, cell_height), Image.Resampling.LANCZOS) + + # Calculate position + x = col * (cell_width + padding) + y = row * (cell_height + padding) + + # Paste into grid + grid.paste(img, (x, y)) + + logging.info( + f"Created {columns}x{rows} grid with {num_images} images ({grid_width}x{grid_height})" + ) + return pil_to_tensor(grid) + + +class MergeImageListsNode(ImageProcessingNode): + """Merge multiple image lists into a single list.""" + + node_id = "MergeImageLists" + search_aliases=["list", "merge list", "make list"] + display_name = "Merge Image Lists (DEPRECATED)" + category = "image/batch" + description = "Concatenate multiple image lists into one." + is_group_process = True # Receives images as list + is_deprecated = True # This node is superseded by the Create List node + + @classmethod + def _group_process(cls, images): + """Simply return the images list (already merged by input handling).""" + # When multiple list inputs are connected, they're concatenated + # For now, this is a simple pass-through + logging.info(f"Merged image list contains {len(images)} images") + return images + + +class MergeTextListsNode(TextProcessingNode): + """Merge multiple text lists into a single list.""" + + node_id = "MergeTextLists" + display_name = "Merge Text Lists (DEPRECATED)" + category = "text" + description = "Concatenate multiple text lists into one." + is_group_process = True # Receives texts as list + is_deprecated = True # This node is superseded by the Create List node + + @classmethod + def _group_process(cls, texts): + """Simply return the texts list (already merged by input handling).""" + # When multiple list inputs are connected, they're concatenated + # For now, this is a simple pass-through + logging.info(f"Merged text list contains {len(texts)} texts") + return texts + + +# ========== Training Dataset Nodes ========== + + +class ResolutionBucket(io.ComfyNode): + """Bucket latents and conditions by resolution for efficient batch training.""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ResolutionBucket", + search_aliases=["bucket by resolution", "group by resolution", "batch by resolution"], + display_name="Resolution Bucket", + category="model/training", + description="Group latents and conditionings into buckets", + is_experimental=True, + is_input_list=True, + inputs=[ + io.Latent.Input( + "latents", + tooltip="List of latent dicts to bucket by resolution.", + ), + io.Conditioning.Input( + "conditioning", + tooltip="List of conditioning lists (must match latents length).", + ), + ], + outputs=[ + io.Latent.Output( + display_name="latents", + is_output_list=True, + tooltip="List of batched latent dicts, one per resolution bucket.", + ), + io.Conditioning.Output( + display_name="conditioning", + is_output_list=True, + tooltip="List of condition lists, one per resolution bucket.", + ), + ], + ) + + @classmethod + def execute(cls, latents, conditioning): + # latents: list[{"samples": tensor}] where tensor is (B, C, H, W), typically B=1 + # conditioning: list[list[cond]] + + # Validate lengths match + if len(latents) != len(conditioning): + raise ValueError( + f"Number of latents ({len(latents)}) does not match number of conditions ({len(conditioning)})." + ) + + # Flatten latents and conditions to individual samples + flat_latents = [] # list of (C, H, W) tensors + flat_conditions = [] # list of condition lists + + for latent_dict, cond in zip(latents, conditioning): + samples = latent_dict["samples"] # (B, C, H, W) + batch_size = samples.shape[0] + + # cond is a list of conditions with length == batch_size + for i in range(batch_size): + flat_latents.append(samples[i]) # (C, H, W) + flat_conditions.append(cond[i]) # single condition + + # Group by resolution (H, W) + buckets = {} # (H, W) -> {"latents": list, "conditions": list} + + for latent, cond in zip(flat_latents, flat_conditions): + # latent shape is (..., H, W) (B, C, H, W) or (B, T, C, H ,W) + h, w = latent.shape[-2], latent.shape[-1] + key = (h, w) + + if key not in buckets: + buckets[key] = {"latents": [], "conditions": []} + + buckets[key]["latents"].append(latent) + buckets[key]["conditions"].append(cond) + + # Convert buckets to output format + output_latents = [] # list[{"samples": tensor}] where tensor is (Bi, ..., H, W) + output_conditions = [] # list[list[cond]] where each inner list has Bi conditions + + for (h, w), bucket_data in buckets.items(): + # Stack latents into batch: list of (..., H, W) -> (Bi, ..., H, W) + stacked_latents = torch.stack(bucket_data["latents"], dim=0) + output_latents.append({"samples": stacked_latents}) + + # Conditions stay as list of condition lists + output_conditions.append(bucket_data["conditions"]) + + logging.info( + f"Resolution bucket ({h}x{w}): {len(bucket_data['latents'])} samples" + ) + + logging.info(f"Created {len(buckets)} resolution buckets from {len(flat_latents)} samples") + return io.NodeOutput(output_latents, output_conditions) + + +class MakeTrainingDataset(io.ComfyNode): + """Encode images with VAE and texts with CLIP to create a training dataset.""" + @classmethod + def define_schema(cls): + return io.Schema( + node_id="MakeTrainingDataset", + search_aliases=["encode dataset"], + display_name="Make Training Dataset", + category="model/training", + description="Encode images with VAE and texts with CLIP to create a training dataset of latents and conditionings.", + is_experimental=True, + is_input_list=True, # images and texts as lists + inputs=[ + io.Image.Input("images", tooltip="List of images to encode."), + io.Vae.Input( + "vae", tooltip="VAE model for encoding images to latents." + ), + io.Clip.Input( + "clip", tooltip="CLIP model for encoding text to conditioning." + ), + io.String.Input( + "texts", + optional=True, + tooltip="List of text captions. Can be length n (matching images), 1 (repeated for all), or omitted (uses empty string).", + force_input=True + ), + ], + outputs=[ + io.Latent.Output( + display_name="latents", + is_output_list=True, + tooltip="List of latent dicts", + ), + io.Conditioning.Output( + display_name="conditioning", + is_output_list=True, + tooltip="List of conditioning lists", + ), + ], + ) + + @classmethod + def execute(cls, images, vae, clip, texts=None): + # Extract scalars (vae and clip are single values wrapped in lists) + vae = vae[0] + clip = clip[0] + + # Handle text list + num_images = len(images) + + if texts is None or len(texts) == 0: + # Treat as [""] for unconditional training + texts = [""] + + if len(texts) == 1 and num_images > 1: + # Repeat single text for all images + texts = texts * num_images + elif len(texts) != num_images: + raise ValueError( + f"Number of texts ({len(texts)}) does not match number of images ({num_images}). " + f"Text list should have length {num_images}, 1, or 0." + ) + + # Encode images with VAE + logging.info(f"Encoding {num_images} images with VAE...") + latents_list = [] # list[{"samples": tensor}] + for img_tensor in images: + # img_tensor is [1, H, W, 3] + latent_tensor = vae.encode(img_tensor[:, :, :, :3]) + latents_list.append({"samples": latent_tensor}) + + # Encode texts with CLIP + logging.info(f"Encoding {len(texts)} texts with CLIP...") + conditioning_list = [] # list[list[cond]] + for text in texts: + if text == "": + cond = clip.encode_from_tokens_scheduled(clip.tokenize("")) + else: + tokens = clip.tokenize(text) + cond = clip.encode_from_tokens_scheduled(tokens) + conditioning_list.append(cond) + + logging.info( + f"Created dataset with {len(latents_list)} latents and {len(conditioning_list)} conditioning." + ) + return io.NodeOutput(latents_list, conditioning_list) + + +class SaveTrainingDataset(io.ComfyNode): + """Save encoded training dataset (latents + conditioning) to disk.""" + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SaveTrainingDataset", + search_aliases=["export dataset", "save dataset"], + display_name="Save Training Dataset", + category="model/training", + description="Save encoded training dataset (latents + conditioning) to disk for efficient loading during training.", + is_experimental=True, + is_output_node=True, + is_input_list=True, # Receive lists + inputs=[ + io.Latent.Input( + "latents", + tooltip="List of latent dicts from MakeTrainingDataset.", + ), + io.Conditioning.Input( + "conditioning", + tooltip="List of conditioning lists from MakeTrainingDataset.", + ), + io.String.Input( + "folder_name", + default="training_dataset", + tooltip="Name of folder to save the dataset into, inside the datasets directory. Subfolders like 'project/run1' are allowed.", + ), + io.Int.Input( + "shard_size", + default=1000, + min=1, + max=100000, + tooltip="Number of samples per shard file.", + advanced=True, + ), + ], + outputs=[], + ) + + @classmethod + def execute(cls, latents, conditioning, folder_name, shard_size): + # Extract scalars + folder_name = folder_name[0] + shard_size = shard_size[0] + + # latents: list[{"samples": tensor}] + # conditioning: list[list[cond]] + + # Validate lengths match + if len(latents) != len(conditioning): + raise ValueError( + f"Number of latents ({len(latents)}) does not match number of conditions ({len(conditioning)}). " + f"Something went wrong in dataset preparation." + ) + + # Create output directory (inside the datasets root, traversal-safe) + output_dir = get_dataset_save_dir(folder_name) + os.makedirs(output_dir, exist_ok=True) + + # Prepare data pairs + num_samples = len(latents) + num_shards = (num_samples + shard_size - 1) // shard_size # Ceiling division + + logging.info( + f"Saving {num_samples} samples to {num_shards} shards in {output_dir}..." + ) + + # Save data in shards + for shard_idx in range(num_shards): + start_idx = shard_idx * shard_size + end_idx = min(start_idx + shard_size, num_samples) + + # Get shard data (list of latent dicts and conditioning lists) + shard_data = { + "latents": latents[start_idx:end_idx], + "conditioning": conditioning[start_idx:end_idx], + } + + # Save shard + shard_filename = f"shard_{shard_idx:04d}.pkl" + shard_path = os.path.join(output_dir, shard_filename) + + with open(shard_path, "wb") as f: + torch.save(shard_data, f) + + logging.info( + f"Saved shard {shard_idx + 1}/{num_shards}: {shard_filename} ({end_idx - start_idx} samples)" + ) + + # Save metadata + metadata = { + "num_samples": num_samples, + "num_shards": num_shards, + "shard_size": shard_size, + } + metadata_path = os.path.join(output_dir, "metadata.json") + with open(metadata_path, "w") as f: + json.dump(metadata, f, indent=2) + + logging.info(f"Successfully saved {num_samples} samples to {output_dir}.") + return io.NodeOutput() + + +class LoadTrainingDataset(io.ComfyNode): + """Load encoded training dataset from disk.""" + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LoadTrainingDataset", + search_aliases=["import dataset", "training data"], + display_name="Load Training Dataset", + category="model/training", + description="Load encoded training dataset (latents + conditioning) from disk for use in training.", + is_experimental=True, + inputs=[ + io.Combo.Input( + "folder_name", + options=list_dataset_folders(), + tooltip="Saved dataset to load, from the datasets directory.", + ), + ], + outputs=[ + io.Latent.Output( + display_name="latents", + is_output_list=True, + tooltip="List of latent dicts", + ), + io.Conditioning.Output( + display_name="conditioning", + is_output_list=True, + tooltip="List of conditioning lists", + ), + ], + ) + + @classmethod + def execute(cls, folder_name): + # Get dataset directory (searched across all dataset roots, traversal-safe) + dataset_dir = get_dataset_dir(folder_name) + + # Find all shard files + shard_files = sorted( + [ + f + for f in os.listdir(dataset_dir) + if f.startswith("shard_") and f.endswith(".pkl") + ] + ) + + if not shard_files: + raise ValueError(f"No shard files found in {dataset_dir}") + + logging.info(f"Loading {len(shard_files)} shards from {dataset_dir}...") + + # Load all shards + all_latents = [] # list[{"samples": tensor}] + all_conditioning = [] # list[list[cond]] + + for shard_file in shard_files: + shard_path = os.path.join(dataset_dir, shard_file) + + with open(shard_path, "rb") as f: + shard_data = torch.load(f, weights_only=True) + + all_latents.extend(shard_data["latents"]) + all_conditioning.extend(shard_data["conditioning"]) + + logging.info(f"Loaded {shard_file}: {len(shard_data['latents'])} samples") + + logging.info( + f"Successfully loaded {len(all_latents)} samples from {dataset_dir}." + ) + return io.NodeOutput(all_latents, all_conditioning) + + +# ========== Extension Setup ========== + + +class DatasetExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + # Data loading/saving nodes + LoadImageDataSetFromFolderNode, + LoadImageTextDataSetFromFolderNode, + SaveImageDataSetToFolderNode, + SaveImageTextDataSetToFolderNode, + # Video data loading nodes + LoadVideoDataSetFromFolderNode, + LoadVideoTextDataSetFromFolderNode, + # Image transform nodes (auto-handle video via per-frame processing) + ResizeImagesByShorterEdgeNode, + ResizeImagesByLongerEdgeNode, + CenterCropImagesNode, + RandomCropImagesNode, + NormalizeImagesNode, + AdjustBrightnessNode, + AdjustContrastNode, + ShuffleDatasetNode, + ShuffleImageTextDatasetNode, + # Video processing nodes (lazy VideoInput in/out) + VideoFrameSampleNode, + VideoTemporalCropNode, + VideoRandomTemporalCropNode, + ShuffleVideoDatasetNode, + ShuffleVideoTextDatasetNode, + # Text transform nodes + TextToLowercaseNode, + TextToUppercaseNode, + TruncateTextNode, + AddTextPrefixNode, + AddTextSuffixNode, + ReplaceTextNode, + StripWhitespaceNode, + # Group processing examples + ImageDeduplicationNode, + ImageGridNode, + MergeImageListsNode, + MergeTextListsNode, + # Training dataset nodes + MakeTrainingDataset, + SaveTrainingDataset, + LoadTrainingDataset, + ResolutionBucket, + ] + + +async def comfy_entrypoint() -> DatasetExtension: + return DatasetExtension() diff --git a/ComfyUI/comfy_extras/nodes_depth_anything_3.py b/ComfyUI/comfy_extras/nodes_depth_anything_3.py new file mode 100644 index 0000000000000000000000000000000000000000..02011251591d2ffa332cc0b023e0173e59f87218 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_depth_anything_3.py @@ -0,0 +1,681 @@ +"""ComfyUI nodes for Depth Anything 3. +Model capability matrix: + +Variant head_type has_sky has_conf cam_dec +DA3-Small dualdpt False True yes +DA3-Base dualdpt False True yes +DA3-Mono-Large dpt True False no +DA3-Metric-Large dpt True False no (raw output is metres) +""" + +from __future__ import annotations + +import logging +from typing_extensions import override + +import torch + +import comfy.model_management as mm +import comfy.sd +import folder_paths +from comfy.ldm.colormap import turbo as _turbo +from comfy.ldm.depth_anything_3 import preprocess as da3_preprocess +from comfy_api.latest import ComfyExtension, Types, io +from comfy.ldm.moge.geometry import triangulate_grid_mesh + +DA3ModelType = io.Custom("DA3_MODEL") +DA3Geometry = io.Custom("DA3_GEOMETRY") +DA3PointCloud = io.Custom("DA3_POINT_CLOUD") + +# DA3_GEOMETRY is a dict with these optional keys (absent when the upstream model didn't produce them): +# +# Per-frame tensors - B = batch size in mono mode; B = S (number of views) in multi-view mode. +# "depth": torch.Tensor (B, H, W) -- raw model depth (always present; matches MoGe convention) +# "image": torch.Tensor (B, H, W, 3) -- source image in [0, 1], CPU (always present) +# "mode": str -- "mono" or "multiview" (always present) +# "sky": torch.Tensor (B, H, W) -- sky probability in [0, 1] (Mono/Metric variants only) +# "confidence": torch.Tensor (B, H, W) -- raw model confidence output (Small/Base variants only) +# +# Multi-view only - S = number of views; the leading 1 is the scene dimension from the model. +# "extrinsics": torch.Tensor (1, S, 3, 4) -- world-to-camera [R|t] matrices +# "intrinsics": torch.Tensor (1, S, 3, 3) -- pixel-space intrinsics +# +# DA3_POINT_CLOUD is a dict: +# "points": torch.Tensor (N, 3) -- 3-D coords in glTF convention (Y-up, Z-back) +# "colors": torch.Tensor (N, 3) -- RGB in [0, 1], or None +# "confidence": torch.Tensor (N,) -- raw confidence per point, or None + + +def _da3_unproject(depth: torch.Tensor, K: torch.Tensor) -> torch.Tensor: + """Pixel-space K⁻¹ unprojection: (H,W) depth → (H,W,3) point map in OpenCV space.""" + H, W = depth.shape + u = torch.arange(W, dtype=torch.float32, device=depth.device) + v = torch.arange(H, dtype=torch.float32, device=depth.device) + u, v = torch.meshgrid(u, v, indexing='xy') # both (H, W) + pix = torch.stack([u, v, torch.ones_like(u)], dim=-1) # (H, W, 3) + rays = torch.einsum('ij,hwj->hwi', torch.linalg.inv(K.to(depth.device)), pix) + return rays * depth.unsqueeze(-1) # (H, W, 3) + + +def _da3_default_K(H: int, W: int) -> torch.Tensor: + """Fallback ~60° FOV pinhole K for mono-mode DA3 (no intrinsics in geometry).""" + fx = fy = float(W) * 0.7 + return torch.tensor([[fx, 0.0, (W - 1) / 2.0], + [0.0, fy, (H - 1) / 2.0], + [0.0, 0.0, 1.0]], dtype=torch.float32) + + +def _da3_get_K(geometry: dict, b: int, H: int, W: int) -> torch.Tensor: + """Return pixel-space K for batch element b, falling back to a default estimate.""" + if "intrinsics" in geometry: + # shape (1, S, 3, 3) - leading scene dimension from the multiview head + return geometry["intrinsics"][0, b].float() + logging.getLogger("comfy").warning( + "DA3_GEOMETRY has no intrinsics (mono-mode model). " + "Using a ~60° FOV estimate; 3-D reconstruction may be inaccurate." + ) + return _da3_default_K(H, W) + + +def _da3_get_extrinsic(geometry: dict, b: int) -> torch.Tensor | None: + """Return the world-to-camera extrinsic for batch element b, or None in mono mode. + + The model outputs (1, S, 3, 4) [R|t] matrices; the fallback identity is (4, 4). + _da3_apply_extrinsic handles both shapes via [:3, :3] / [:3, 3] slicing. + """ + if "extrinsics" not in geometry: + return None + return geometry["extrinsics"][0, b].float() + + +def _da3_apply_extrinsic(points_cam: torch.Tensor, E: torch.Tensor) -> torch.Tensor: + """Transform (H,W,3) OpenCV camera-space points to world space.""" + E = E.to(points_cam.device).float() + if not torch.isfinite(E).all(): + logging.getLogger("comfy").warning( + "DA3 extrinsic matrix contains non-finite values (pose estimation may have failed). " + "Falling back to camera-space coordinates." + ) + return points_cam + H, W, _ = points_cam.shape + R = E[:3, :3] # (3, 3) rotation + t = E[:3, 3] # (3,) translation + R_inv = R.T # rotation inverse = transpose for orthogonal R + t_inv = -(R_inv @ t) # (3,) + pts = points_cam.reshape(-1, 3) # (N, 3) + pts_world = pts @ R_inv.T + t_inv # (N, 3) + return pts_world.reshape(H, W, 3) + + +def _normalize_confidence(conf: torch.Tensor) -> torch.Tensor: + """Map raw confidence to [0, 1] per image.""" + B = conf.shape[0] + out = [] + for i in range(B): + c = conf[i] + c_min, c_max = c.min(), c.max() + out.append((c - c_min) / (c_max - c_min) if c_max > c_min else torch.ones_like(c)) + return torch.stack(out, dim=0) + + +def _da3_build_mask(geometry: dict, b: int, H: int, W: int, confidence_threshold: float, use_sky_mask: bool) -> torch.Tensor: + """Build (H,W) bool keep-mask from sky probability and confidence.""" + mask = torch.ones(H, W, dtype=torch.bool) + if use_sky_mask and "sky" in geometry: + mask = mask & (geometry["sky"][b] < 0.5) + if "confidence" in geometry and confidence_threshold > 0.0: + conf_norm = _normalize_confidence(geometry["confidence"][b:b + 1])[0] + mask = mask & (conf_norm >= confidence_threshold) + return mask + + +class LoadDA3Model(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LoadDA3Model", + display_name="Load Depth Anything 3", + category="model/loaders", + inputs=[ + io.Combo.Input( + "model_name", + options=folder_paths.get_filename_list("geometry_estimation"), + ), + io.Combo.Input( + "weight_dtype", + options=["default", "fp16", "bf16", "fp32"], + default="default", + ), + ], + outputs=[DA3ModelType.Output()], + ) + + @classmethod + def execute(cls, model_name, weight_dtype) -> io.NodeOutput: + model_options = {} + if weight_dtype == "fp16": + model_options["dtype"] = torch.float16 + elif weight_dtype == "bf16": + model_options["dtype"] = torch.bfloat16 + elif weight_dtype == "fp32": + model_options["dtype"] = torch.float32 + + path = folder_paths.get_full_path_or_raise("geometry_estimation", model_name) + model = comfy.sd.load_diffusion_model(path, model_options=model_options) + return io.NodeOutput(model) + + +def _run_da3(model_patcher, image: torch.Tensor, process_res: int, method: str = "upper_bound_resize"): + """Run DA3 on (B,H,W,3), returns depth/conf/sky at original resolution (or None).""" + assert image.ndim == 4 and image.shape[-1] == 3, f"expected (B,H,W,3) IMAGE; got {tuple(image.shape)}" + + B, H, W, _ = image.shape + mm.load_model_gpu(model_patcher) + diffusion = model_patcher.model.diffusion_model + device = mm.get_torch_device() + dtype = diffusion.dtype if diffusion.dtype is not None else torch.float32 + + depths, confs, skies = [], [], [] + for i in range(B): + single = image[i:i + 1].to(device) + x = da3_preprocess.preprocess_image(single, process_res=process_res, method=method) + x = x.to(dtype=dtype) + with torch.no_grad(): + out = diffusion(x) + + depth_lr = out["depth"] + depth_full = torch.nn.functional.interpolate( + depth_lr.unsqueeze(1).float(), size=(H, W), + mode="bilinear", align_corners=False, + ).squeeze(1).cpu() + depths.append(depth_full) + + if "depth_conf" in out: + conf_full = torch.nn.functional.interpolate( + out["depth_conf"].unsqueeze(1).float(), size=(H, W), + mode="bilinear", align_corners=False, + ).squeeze(1).cpu() + confs.append(conf_full) + if "sky" in out: + sky_full = torch.nn.functional.interpolate( + out["sky"].unsqueeze(1).float(), size=(H, W), + mode="bilinear", align_corners=False, + ).squeeze(1).cpu() + skies.append(sky_full) + + depth = torch.cat(depths, dim=0) + confidence = torch.cat(confs, dim=0) if confs else None + sky = torch.cat(skies, dim=0) if skies else None + return depth, confidence, sky + + +class DA3Inference(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="DA3Inference", + search_aliases=["depth", "geometry", "da3", "depth anything", "monocular", "pointmap", "sky", "3d", "metric depth", "disparity"], + display_name="Run Depth Anything 3", + category="image/geometry estimation", + description="Run Depth Anything 3 on an image. In multi-view mode each image is treated as a separate view of the same scene.", + inputs=[ + DA3ModelType.Input("da3_model"), + io.Image.Input("image"), + io.Int.Input("resolution", default=504, min=140, max=2520, step=14, + tooltip="Resolution the model runs at (longest side, multiple of 14).\n" + "Lower = faster / less VRAM.\n" + "Higher = more detail.\n" + "Output is upsampled back to the original size."), + io.Combo.Input("resize_method", options=["upper_bound_resize", "lower_bound_resize"], default="upper_bound_resize", + tooltip="upper_bound_resize: scale so the longest side = resolution (caps memory, default).\n" + "lower_bound_resize: scale so the shortest side = resolution (preserves more detail on tall/wide images, uses more memory)."), + io.DynamicCombo.Input("mode", tooltip="mono: single view image (works with any model variant).\n" + "multiview: all images processed together for geometric consistency + camera pose (for Small/Base models only).", + options=[ + io.DynamicCombo.Option("mono", []), + io.DynamicCombo.Option("multiview", [ + io.Combo.Input("ref_view_strategy", options=["saddle_balanced", "saddle_sim_range", "first", "middle"], default="saddle_balanced", + tooltip="Which view acts as the geometric anchor.\n" + "- saddle_balanced: the view most 'average' across all others (best general choice).\n" + "- saddle_sim_range: the view most visually distinct from the others.\n" + "- first / middle: fixed positional picks."), + io.Combo.Input("pose_method", options=["cam_dec", "ray_pose"], default="cam_dec", + tooltip="How the camera field-of-view is estimated (for Small/Base models only).\n" + "- cam_dec: learned from image features.\n" + "- ray_pose: derived geometrically from the model's 3D ray output.\n" + "Affects perspective correctness of the 3D output. Try both if results look distorted."), + ]), + ]), + ], + outputs=[ + DA3Geometry.Output("da3_geometry", tooltip="Dictionary of non-normalized tensors.\n" + "Always has the keys: depth, image, mode.\n" + "Optional keys: sky (for Mono/Metric), confidence (for Small/Base), extrinsics + intrinsics (for multi-view)."), + ], + ) + + @classmethod + def execute(cls, da3_model, image, resolution, resize_method, mode) -> io.NodeOutput: + mode_val = mode["mode"] # "mono" or "multiview" + + if mode_val == "mono": + return cls._execute_mono(da3_model, image, resolution, resize_method) + + # Capability checks for multi-view mode. + diffusion = da3_model.model.diffusion_model + pose_method = mode["pose_method"] + ref_view_strategy = mode["ref_view_strategy"] + + has_cam_dec = diffusion.cam_dec is not None + has_dualdpt = diffusion.head_type == "dualdpt" + + if not has_cam_dec and not has_dualdpt: + raise ValueError( + "multi-view mode requires Small or Base model. The loaded model " + f"(head_type='{diffusion.head_type}') does not support cross-view " + "attention or camera pose estimation. Switch mode to 'mono', or " + "load Small or Base model for mult-view." + ) + + if pose_method == "cam_dec" and not has_cam_dec: + raise ValueError( + "pose_method='cam_dec' requires a camera decoder, but the loaded " + f"model (head_type='{diffusion.head_type}') does not have one. " + "Use pose_method='ray_pose' instead." + ) + if pose_method == "ray_pose" and not has_dualdpt: + raise ValueError( + "pose_method='ray_pose' requires a DualDPT head, but the loaded " + f"model has a '{diffusion.head_type}' head. " + "Use pose_method='cam_dec' instead." + ) + + return cls._execute_multiview( + da3_model, image, resolution, resize_method, + ref_view_strategy, pose_method, + ) + + @classmethod + def _execute_mono(cls, model, image, resolution, resize_method) -> io.NodeOutput: + depth, confidence, sky = _run_da3(model, image, resolution, method=resize_method) + + geometry: dict = { + "depth": depth.contiguous(), + "image": image[..., :3].cpu(), + "mode": "mono", + } + if sky is not None: + geometry["sky"] = sky.contiguous() + if confidence is not None: + geometry["confidence"] = confidence.contiguous() + return io.NodeOutput(geometry) + + @classmethod + def _execute_multiview(cls, model, image, resolution, resize_method, ref_view_strategy, pose_method) -> io.NodeOutput: + assert image.ndim == 4 and image.shape[-1] == 3, \ + f"expected (B,H,W,3) IMAGE; got {tuple(image.shape)}" + S, H, W, _ = image.shape + + mm.load_model_gpu(model) + diffusion = model.model.diffusion_model + device = mm.get_torch_device() + dtype = diffusion.dtype if diffusion.dtype is not None else torch.float32 + + # All views in a single forward pass: (1, S, 3, H', W'). + x = image.to(device) + x = da3_preprocess.preprocess_image(x, process_res=resolution, method=resize_method) + x = x.to(dtype=dtype).unsqueeze(0) + + use_ray_pose = (pose_method == "ray_pose") + with torch.no_grad(): + out = diffusion(x, use_ray_pose=use_ray_pose, ref_view_strategy=ref_view_strategy) + + depth = torch.nn.functional.interpolate( + out["depth"].float().unsqueeze(1), size=(H, W), + mode="bilinear", align_corners=False, + ).squeeze(1).cpu() + + sky = None + if "sky" in out: + sky = torch.nn.functional.interpolate( + out["sky"].unsqueeze(1).float(), size=(H, W), + mode="bilinear", align_corners=False, + ).squeeze(1).cpu() + + if "extrinsics" in out and "intrinsics" in out: + extrinsics = out["extrinsics"].float().cpu() + intrinsics = out["intrinsics"].float().cpu() + else: + extrinsics = torch.eye(4)[None, None].expand(1, S, 4, 4).clone() + intrinsics = torch.eye(3)[None, None].expand(1, S, 3, 3).clone() + + geometry: dict = { + "depth": depth.contiguous(), + "image": image[..., :3].cpu(), + "mode": "multiview", + "extrinsics": extrinsics.contiguous(), + "intrinsics": intrinsics.contiguous(), + } + if sky is not None: + geometry["sky"] = sky.contiguous() + if "depth_conf" in out: + conf = torch.nn.functional.interpolate( + out["depth_conf"].unsqueeze(1).float(), size=(H, W), + mode="bilinear", align_corners=False, + ).squeeze(1).cpu() + geometry["confidence"] = conf.contiguous() + return io.NodeOutput(geometry) + + +class DA3Render(io.ComfyNode): + """Render a visualization from a DA3_GEOMETRY packet.""" + + _DEPTH_RENDER_INPUTS = [ + io.Combo.Input("normalization", + options=["v2_style", "min_max", "raw"], + default="v2_style", + tooltip="- v2_style: mean/std normalisation for perceptually balanced results (default).\n" + "- min_max: stretches the full depth range to [0, 1] for maximum contrast.\n" + "- raw: no scaling,preserves metric units for Metric model."), + io.Boolean.Input("apply_sky_clip", default=False, + tooltip="Clip sky-region depth to the 99th percentile of foreground depth before normalisation. " + "Requires a sky key in the da3_geometry input (for Mono/Metric models only)."), + ] + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="DA3Render", + display_name="Render Depth Anything 3", + category="image/geometry estimation", + description="Render a depth map, confidence map, or sky mask from Depth Anything 3 geometry data.", + inputs=[ + DA3Geometry.Input("da3_geometry"), + io.DynamicCombo.Input("output", + tooltip="- depth: normalised greyscale depth image.\n" + "- depth_colored: depth mapped through the Turbo colormap.\n" + "- sky_mask: sky probability in [0, 1] (for Mono/Metric models only).\n" + "- confidence: normalised depth confidence (for Small/Base models only).", + options=[ + io.DynamicCombo.Option("depth", cls._DEPTH_RENDER_INPUTS), + io.DynamicCombo.Option("depth_colored", cls._DEPTH_RENDER_INPUTS), + io.DynamicCombo.Option("sky_mask", [ + io.Boolean.Input("colored", default=False, tooltip="Apply the Turbo colormap to the sky mask."), + ]), + io.DynamicCombo.Option("confidence", [ + io.Boolean.Input("colored", default=False, tooltip="Apply the Turbo colormap to the confidence map."), + ]), + ]), + ], + outputs=[io.Image.Output()], + ) + + @classmethod + def execute(cls, da3_geometry, output) -> io.NodeOutput: + output_val = output["output"] + + if output_val in ("depth", "depth_colored"): + normalization = output["normalization"] + apply_sky_clip = output["apply_sky_clip"] + if apply_sky_clip and "sky" not in da3_geometry: + raise ValueError( + "apply_sky_clip=True requires a sky tensor in the da3_geometry input, but none is present. " + "Run with Mono/Metric models or set apply_sky_clip=False." + ) + depth = da3_geometry["depth"] + sky = da3_geometry.get("sky") + if apply_sky_clip and sky is not None: + depth = torch.stack([ + da3_preprocess.apply_sky_aware_clip(depth[i], sky[i]) + for i in range(depth.shape[0]) + ], dim=0) + grey = cls._depth_to_image(depth, sky, normalization) # (B,H,W,3) greyscale + result = _turbo(grey[..., 0]) if output_val == "depth_colored" else grey + + elif output_val == "sky_mask": + if "sky" not in da3_geometry: + raise ValueError("geometry has no sky output; run with Mono/Metric models.") + sky = da3_geometry["sky"] + if output["colored"]: + result = _turbo(sky) + else: + result = sky.unsqueeze(-1).expand(*sky.shape, 3).contiguous() + + elif output_val == "confidence": + if "confidence" not in da3_geometry: + raise ValueError("da3_geometry has no confidence output; run with Small/Base models.") + conf = _normalize_confidence(da3_geometry["confidence"]) + if output["colored"]: + result = _turbo(conf) + else: + result = conf.unsqueeze(-1).expand(*conf.shape, 3).contiguous() + + else: + raise ValueError(f"Unknown output mode: {output_val}") + + return io.NodeOutput(result.float()) + + @staticmethod + def _depth_to_image(depth: torch.Tensor, sky_for_norm: torch.Tensor | None, normalization: str) -> torch.Tensor: + """Normalise depth and pack as an (B,H,W,3) image tensor.""" + + N = depth.shape[0] + if normalization == "v2_style": + norm = torch.stack([ + da3_preprocess.normalize_depth_v2_style( + depth[i], sky_for_norm[i] if sky_for_norm is not None else None) + for i in range(N) + ], dim=0) + elif normalization == "min_max": + norm = da3_preprocess.normalize_depth_min_max(depth) + else: + norm = depth + + out = norm.unsqueeze(-1).repeat(1, 1, 1, 3) + if normalization != "raw": + out = out.clamp(0.0, 1.0) + return out.contiguous() + + +class DA3GeometryToMesh(io.ComfyNode): + """Convert a DA3_GEOMETRY packet into a Types.MESH by unprojecting depth and triangulating.""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="DA3GeometryToMesh", + search_aliases=["da3", "depth anything", "mesh", "geometry", "3d", "triangulate"], + display_name="Convert DA3 Geometry to Mesh", + category="image/geometry estimation", + description="Convert a depth map into a triangulated 3D mesh.", + inputs=[ + DA3Geometry.Input("da3_geometry"), + io.Int.Input("batch_index", default=0, min=0, max=4096, tooltip="Which image of a batch to convert. Per-image vertex counts differ so batches cannot be stacked."), + io.Int.Input("decimation", default=1, min=1, max=8, tooltip="Vertex stride. 1 = full resolution, 2 = half, etc."), + io.Float.Input("discontinuity_threshold", default=0.04, min=0.0, max=1.0, step=0.01, tooltip="Drop triangles whose 3x3 depth span exceeds this fraction. 0 = off."), + io.Float.Input("confidence_threshold", default=0.1, min=0.0, max=1.0, step=0.01, + tooltip="Exclude pixels whose per-image normalised confidence is below this value (0 = keep all, 1 = keep only the single most confident pixel). " + "Used when the geometry has a confidence map (Small/Base models)."), + io.Boolean.Input("use_sky_mask", default=True, tooltip="Exclude sky-probability pixels (sky >= 0.5) from the mesh. Used when the geometry has a sky map (Mono/Metric models)."), + io.Boolean.Input("texture", default=True, tooltip="Use the source image as a base color texture."), + ], + outputs=[io.Mesh.Output()], + ) + + @classmethod + def execute(cls, da3_geometry, batch_index, decimation, discontinuity_threshold, confidence_threshold, use_sky_mask, texture) -> io.NodeOutput: + depth_all = da3_geometry["depth"] # (B, H, W) + B = depth_all.shape[0] + if batch_index >= B: + raise ValueError(f"batch_index {batch_index} is out of range; DA3_GEOMETRY has batch size {B}.") + + depth = depth_all[batch_index] # (H, W) + H, W = depth.shape + + # NaN/inf depth would propagate silently through unproject and produce an + # empty mesh; replace them with 0 here so those pixels are later excluded + # by the isfinite check inside triangulate_grid_mesh. + depth = depth.clone() + n_bad = (~torch.isfinite(depth)).sum().item() + if n_bad: + logging.getLogger("comfy").warning( + f"DA3GeometryToMesh: depth[{batch_index}] has {n_bad} non-finite pixels " + f"({100*n_bad/(H*W):.1f}%) - zeroed before unproject." + ) + depth[~torch.isfinite(depth)] = 0.0 + logging.getLogger("comfy").debug( + f"DA3GeometryToMesh: depth[{batch_index}] range " + f"[{depth.min():.4g}, {depth.max():.4g}], mean={depth.mean():.4g}" + ) + + K = _da3_get_K(da3_geometry, batch_index, H, W) + points = _da3_unproject(depth, K) # (H, W, 3) in OpenCV camera space + + # Apply world-to-camera inverse so multi-view frames share a common world frame. + E = _da3_get_extrinsic(da3_geometry, batch_index) + if E is not None: + points = _da3_apply_extrinsic(points, E) + + # Mask invalid pixels by setting them to inf so triangulate_grid_mesh skips them. + mask = _da3_build_mask(da3_geometry, batch_index, H, W, confidence_threshold, use_sky_mask) + # Also exclude pixels where depth was invalid. + mask = mask & (depth_all[batch_index] > 0) & torch.isfinite(depth_all[batch_index]) + points = points.clone() + points[~mask] = float('inf') + + verts, faces, uvs = triangulate_grid_mesh( + points, + decimation=decimation, + discontinuity_threshold=discontinuity_threshold, + depth=depth, + ) + if verts.shape[0] == 0 or faces.shape[0] == 0: + raise ValueError( + "DA3GeometryToMesh produced an empty mesh. " + "Try raising discontinuity_threshold, lowering confidence_threshold, " + "or disabling use_sky_mask." + ) + + # OpenCV (X right, Y down, Z forward) → glTF (X right, Y up, Z back). + # Same transform as MoGePointMapToMesh perspective branch. + verts = verts * torch.tensor([1.0, -1.0, -1.0], dtype=verts.dtype) + faces = faces[:, [0, 2, 1]].contiguous() + + tex = da3_geometry["image"][batch_index:batch_index + 1] if texture else None + mesh = Types.MESH( + vertices=verts.unsqueeze(0), + faces=faces.unsqueeze(0), + uvs=uvs.unsqueeze(0), + texture=tex, + ) + return io.NodeOutput(mesh) + + +class DA3GeometryToPointCloud(io.ComfyNode): + """Unproject a DA3_GEOMETRY depth map into a filtered DA3_POINT_CLOUD.""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="DA3GeometryToPointCloud", + search_aliases=["da3", "depth anything", "point cloud", "pointcloud", "3d", "geometry"], + display_name="Convert DA3 Geometry to Point Cloud", + category="image/geometry estimation", + description="Convert a depth map into a 3D point cloud.", + inputs=[ + DA3Geometry.Input("da3_geometry"), + io.Int.Input("batch_index", default=0, min=0, max=4096, tooltip="Which image of a batch to convert."), + io.Float.Input("confidence_threshold", default=0.1, min=0.0, max=1.0, step=0.01, + tooltip="Exclude pixels whose per-image normalised confidence is below this value (0 = keep all). Used when the geometry has a confidence map (Small/Base models)."), + io.Boolean.Input("use_sky_mask", default=True, + tooltip="Exclude sky-probability pixels (sky >= 0.5). Used when the geometry has a sky map (Mono/Metric models)."), + io.Int.Input("downsample", default=1, min=1, max=16, + tooltip="Take every Nth pixel (1 = full resolution). Higher values give fewer points and faster processing."), + ], + # TODO: add a proper PointCloud output type + outputs=[DA3PointCloud.Output(display_name="point_cloud")], + ) + + @classmethod + def execute(cls, da3_geometry, batch_index, confidence_threshold, use_sky_mask, downsample) -> io.NodeOutput: + depth_all = da3_geometry["depth"] # (B, H, W) + B = depth_all.shape[0] + if batch_index >= B: + raise ValueError(f"batch_index {batch_index} is out of range; DA3_GEOMETRY has batch size {B}.") + + depth = depth_all[batch_index].clone() # (H, W) + depth[~torch.isfinite(depth)] = 0.0 + H, W = depth.shape + + K = _da3_get_K(da3_geometry, batch_index, H, W) + + if downsample > 1: + depth = depth[::downsample, ::downsample].contiguous() + # Scale intrinsics to the downsampled grid. + K = K.clone() + K[0, :] /= downsample + K[1, :] /= downsample + + H_ds, W_ds = depth.shape + points = _da3_unproject(depth, K) # (H_ds, W_ds, 3) in OpenCV camera space + + # Apply world-to-camera inverse so multi-view frames share a common world frame. + E = _da3_get_extrinsic(da3_geometry, batch_index) + if E is not None: + points = _da3_apply_extrinsic(points, E) + + # Rebuild mask at downsampled resolution. + mask = _da3_build_mask(da3_geometry, batch_index, H, W, confidence_threshold, use_sky_mask) + if downsample > 1: + mask = mask[::downsample, ::downsample] + + mask = mask & torch.isfinite(depth) + + # OpenCV → glTF: flip Y and Z. + points_gltf = points.clone() + points_gltf[..., 1] *= -1.0 + points_gltf[..., 2] *= -1.0 + + pts_flat = points_gltf.reshape(-1, 3)[mask.reshape(-1)] + + colors_flat = None + if "image" in da3_geometry: + img = da3_geometry["image"][batch_index] # (H, W, 3) + if downsample > 1: + img = img[::downsample, ::downsample] + colors_flat = img.reshape(-1, 3)[mask.reshape(-1)] + + conf_flat = None + if "confidence" in da3_geometry: + conf = da3_geometry["confidence"][batch_index] # (H, W) + if downsample > 1: + conf = conf[::downsample, ::downsample] + conf_flat = conf.reshape(-1)[mask.reshape(-1)] + + if pts_flat.shape[0] == 0: + raise ValueError( + "DA3GeometryToPointCloud produced zero points after filtering. " + "Try lowering confidence_threshold or disabling use_sky_mask." + ) + + return io.NodeOutput({ + "points": pts_flat, + "colors": colors_flat, + "confidence": conf_flat, + }) + + +class DA3Extension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + LoadDA3Model, + DA3Inference, + DA3Render, + DA3GeometryToMesh, + # DA3GeometryToPointCloud, # Keep this commented out for now until we have a proper PointCloud output type + ] + + +async def comfy_entrypoint() -> DA3Extension: + return DA3Extension() diff --git a/ComfyUI/comfy_extras/nodes_differential_diffusion.py b/ComfyUI/comfy_extras/nodes_differential_diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..4fa61ad0e5be031a1dd8cc08ee6bb1e7c040df48 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_differential_diffusion.py @@ -0,0 +1,73 @@ +# code adapted from https://github.com/exx8/differential-diffusion + +from typing_extensions import override + +import torch +from comfy_api.latest import ComfyExtension, io + + +class DifferentialDiffusion(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="DifferentialDiffusion", + search_aliases=["inpaint gradient", "variable denoise strength"], + display_name="Differential Diffusion", + category="experimental", + inputs=[ + io.Model.Input("model"), + io.Float.Input( + "strength", + default=1.0, + min=0.0, + max=1.0, + step=0.01, + optional=True, + ), + ], + outputs=[io.Model.Output()], + is_experimental=True, + ) + + @classmethod + def execute(cls, model, strength=1.0) -> io.NodeOutput: + model = model.clone() + model.set_model_denoise_mask_function(lambda *args, **kwargs: cls.forward(*args, **kwargs, strength=strength)) + return io.NodeOutput(model) + + @classmethod + def forward(cls, sigma: torch.Tensor, denoise_mask: torch.Tensor, extra_options: dict, strength: float): + model = extra_options["model"] + step_sigmas = extra_options["sigmas"] + sigma_to = model.inner_model.model_sampling.sigma_min + if step_sigmas[-1] > sigma_to: + sigma_to = step_sigmas[-1] + sigma_from = step_sigmas[0] + + ts_from = model.inner_model.model_sampling.timestep(sigma_from) + ts_to = model.inner_model.model_sampling.timestep(sigma_to) + current_ts = model.inner_model.model_sampling.timestep(sigma[0]) + + threshold = (current_ts - ts_to) / (ts_from - ts_to) + + # Generate the binary mask based on the threshold + binary_mask = (denoise_mask >= threshold).to(denoise_mask.dtype) + + # Blend binary mask with the original denoise_mask using strength + if strength and strength < 1: + blended_mask = strength * binary_mask + (1 - strength) * denoise_mask + return blended_mask + else: + return binary_mask + + +class DifferentialDiffusionExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + DifferentialDiffusion, + ] + + +async def comfy_entrypoint() -> DifferentialDiffusionExtension: + return DifferentialDiffusionExtension() diff --git a/ComfyUI/comfy_extras/nodes_easycache.py b/ComfyUI/comfy_extras/nodes_easycache.py new file mode 100644 index 0000000000000000000000000000000000000000..9e907d371c56cab1793b28232b241327527af44e --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_easycache.py @@ -0,0 +1,530 @@ +from __future__ import annotations +from typing import TYPE_CHECKING, Union +from comfy_api.latest import io, ComfyExtension +import comfy.patcher_extension +import logging +import torch +import comfy.model_patcher +if TYPE_CHECKING: + from uuid import UUID + + +def _extract_tensor(data, output_channels): + """Extract tensor from data, handling both single tensors and lists.""" + if isinstance(data, list): + # LTX2 AV tensors: [video, audio] + return data[0][:, :output_channels], data[1][:, :output_channels] + return data[:, :output_channels], None + + +def easycache_forward_wrapper(executor, *args, **kwargs): + # get values from args + transformer_options: dict[str] = args[-1] + if not isinstance(transformer_options, dict): + transformer_options = kwargs.get("transformer_options") + if not transformer_options: + transformer_options = args[-2] + easycache: EasyCacheHolder = transformer_options["easycache"] + x, ax = _extract_tensor(args[0], easycache.output_channels) + sigmas = transformer_options["sigmas"] + uuids = transformer_options["uuids"] + if sigmas is not None and easycache.is_past_end_timestep(sigmas): + return executor(*args, **kwargs) + # prepare next x_prev + has_first_cond_uuid = easycache.has_first_cond_uuid(uuids) + next_x_prev = x + input_change = None + do_easycache = easycache.should_do_easycache(sigmas) + if do_easycache: + easycache.check_metadata(x) + # if there isn't a cache diff for current conds, we cannot skip this step + can_apply_cache_diff = easycache.can_apply_cache_diff(uuids) + # if first cond marked this step for skipping, skip it and use appropriate cached values + if easycache.skip_current_step and can_apply_cache_diff: + if easycache.verbose: + logging.info(f"EasyCache [verbose] - was marked to skip this step by {easycache.first_cond_uuid}. Present uuids: {uuids}") + result = easycache.apply_cache_diff(x, uuids) + if ax is not None: + result_audio = easycache.apply_cache_diff(ax, uuids, is_audio=True) + return [result, result_audio] + return result + if easycache.initial_step: + easycache.first_cond_uuid = uuids[0] + has_first_cond_uuid = easycache.has_first_cond_uuid(uuids) + easycache.initial_step = False + if has_first_cond_uuid: + if easycache.has_x_prev_subsampled(): + input_change = (easycache.subsample(x, uuids, clone=False) - easycache.x_prev_subsampled).flatten().abs().mean() + if easycache.has_output_prev_norm() and easycache.has_relative_transformation_rate(): + approx_output_change_rate = (easycache.relative_transformation_rate * input_change) / easycache.output_prev_norm + easycache.cumulative_change_rate += approx_output_change_rate + if easycache.cumulative_change_rate < easycache.reuse_threshold and can_apply_cache_diff: + if easycache.verbose: + logging.info(f"EasyCache [verbose] - skipping step; cumulative_change_rate: {easycache.cumulative_change_rate}, reuse_threshold: {easycache.reuse_threshold}") + # other conds should also skip this step, and instead use their cached values + easycache.skip_current_step = True + result = easycache.apply_cache_diff(x, uuids) + if ax is not None: + result_audio = easycache.apply_cache_diff(ax, uuids, is_audio=True) + return [result, result_audio] + return result + else: + if easycache.verbose: + logging.info(f"EasyCache [verbose] - NOT skipping step; cumulative_change_rate: {easycache.cumulative_change_rate}, reuse_threshold: {easycache.reuse_threshold}") + easycache.cumulative_change_rate = 0.0 + + full_output: torch.Tensor = executor(*args, **kwargs) + output, audio_output = _extract_tensor(full_output, easycache.output_channels) + if has_first_cond_uuid and easycache.has_output_prev_norm(): + output_change = (easycache.subsample(output, uuids, clone=False) - easycache.output_prev_subsampled).flatten().abs().mean() + if easycache.verbose: + output_change_rate = output_change / easycache.output_prev_norm + easycache.output_change_rates.append(output_change_rate.item()) + if easycache.has_relative_transformation_rate(): + approx_output_change_rate = (easycache.relative_transformation_rate * input_change) / easycache.output_prev_norm + easycache.approx_output_change_rates.append(approx_output_change_rate.item()) + if easycache.verbose: + logging.info(f"EasyCache [verbose] - approx_output_change_rate: {approx_output_change_rate}") + if input_change is not None: + easycache.relative_transformation_rate = output_change / input_change + if easycache.verbose: + logging.info(f"EasyCache [verbose] - output_change_rate: {output_change_rate}") + # TODO: allow cache_diff to be offloaded + easycache.update_cache_diff(output, next_x_prev, uuids) + if audio_output is not None: + easycache.update_cache_diff(audio_output, ax, uuids, is_audio=True) + if has_first_cond_uuid: + easycache.x_prev_subsampled = easycache.subsample(next_x_prev, uuids) + easycache.output_prev_subsampled = easycache.subsample(output, uuids) + easycache.output_prev_norm = output.flatten().abs().mean() + if easycache.verbose: + logging.info(f"EasyCache [verbose] - x_prev_subsampled: {easycache.x_prev_subsampled.shape}") + return full_output + +def lazycache_predict_noise_wrapper(executor, *args, **kwargs): + # get values from args + timestep: float = args[1] + model_options: dict[str] = args[2] + easycache: LazyCacheHolder = model_options["transformer_options"]["easycache"] + if easycache.is_past_end_timestep(timestep): + return executor(*args, **kwargs) + x: torch.Tensor = args[0][:, :easycache.output_channels] + # prepare next x_prev + next_x_prev = x + input_change = None + do_easycache = easycache.should_do_easycache(timestep) + if do_easycache: + easycache.check_metadata(x) + if easycache.has_x_prev_subsampled(): + if easycache.has_x_prev_subsampled(): + input_change = (easycache.subsample(x, clone=False) - easycache.x_prev_subsampled).flatten().abs().mean() + if easycache.has_output_prev_norm() and easycache.has_relative_transformation_rate(): + approx_output_change_rate = (easycache.relative_transformation_rate * input_change) / easycache.output_prev_norm + easycache.cumulative_change_rate += approx_output_change_rate + if easycache.cumulative_change_rate < easycache.reuse_threshold: + if easycache.verbose: + logging.info(f"LazyCache [verbose] - skipping step; cumulative_change_rate: {easycache.cumulative_change_rate}, reuse_threshold: {easycache.reuse_threshold}") + # other conds should also skip this step, and instead use their cached values + easycache.skip_current_step = True + return easycache.apply_cache_diff(x) + else: + if easycache.verbose: + logging.info(f"LazyCache [verbose] - NOT skipping step; cumulative_change_rate: {easycache.cumulative_change_rate}, reuse_threshold: {easycache.reuse_threshold}") + easycache.cumulative_change_rate = 0.0 + output: torch.Tensor = executor(*args, **kwargs) + if easycache.has_output_prev_norm(): + output_change = (easycache.subsample(output, clone=False) - easycache.output_prev_subsampled).flatten().abs().mean() + if easycache.verbose: + output_change_rate = output_change / easycache.output_prev_norm + easycache.output_change_rates.append(output_change_rate.item()) + if easycache.has_relative_transformation_rate(): + approx_output_change_rate = (easycache.relative_transformation_rate * input_change) / easycache.output_prev_norm + easycache.approx_output_change_rates.append(approx_output_change_rate.item()) + if easycache.verbose: + logging.info(f"LazyCache [verbose] - approx_output_change_rate: {approx_output_change_rate}") + if input_change is not None: + easycache.relative_transformation_rate = output_change / input_change + if easycache.verbose: + logging.info(f"LazyCache [verbose] - output_change_rate: {output_change_rate}") + # TODO: allow cache_diff to be offloaded + easycache.update_cache_diff(output, next_x_prev) + easycache.x_prev_subsampled = easycache.subsample(next_x_prev) + easycache.output_prev_subsampled = easycache.subsample(output) + easycache.output_prev_norm = output.flatten().abs().mean() + if easycache.verbose: + logging.info(f"LazyCache [verbose] - x_prev_subsampled: {easycache.x_prev_subsampled.shape}") + return output + +def easycache_calc_cond_batch_wrapper(executor, *args, **kwargs): + model_options = args[-1] + easycache: EasyCacheHolder = model_options["transformer_options"]["easycache"] + easycache.skip_current_step = False + # TODO: check if first_cond_uuid is active at this timestep; otherwise, EasyCache needs to be partially reset + return executor(*args, **kwargs) + +def easycache_sample_wrapper(executor, *args, **kwargs): + """ + This OUTER_SAMPLE wrapper makes sure easycache is prepped for current run, and all memory usage is cleared at the end. + """ + try: + guider = executor.class_obj + orig_model_options = guider.model_options + guider.model_options = comfy.model_patcher.create_model_options_clone(orig_model_options) + # clone and prepare timesteps + guider.model_options["transformer_options"]["easycache"] = guider.model_options["transformer_options"]["easycache"].clone().prepare_timesteps(guider.model_patcher.model.model_sampling) + easycache: Union[EasyCacheHolder, LazyCacheHolder] = guider.model_options['transformer_options']['easycache'] + logging.info(f"{easycache.name} enabled - threshold: {easycache.reuse_threshold}, start_percent: {easycache.start_percent}, end_percent: {easycache.end_percent}") + return executor(*args, **kwargs) + finally: + easycache = guider.model_options['transformer_options']['easycache'] + output_change_rates = easycache.output_change_rates + approx_output_change_rates = easycache.approx_output_change_rates + if easycache.verbose: + logging.info(f"{easycache.name} [verbose] - output_change_rates {len(output_change_rates)}: {output_change_rates}") + logging.info(f"{easycache.name} [verbose] - approx_output_change_rates {len(approx_output_change_rates)}: {approx_output_change_rates}") + total_steps = len(args[3])-1 + # catch division by zero for log statement; sucks to crash after all sampling is done + try: + speedup = total_steps/(total_steps-easycache.total_steps_skipped) + except ZeroDivisionError: + speedup = 1.0 + logging.info(f"{easycache.name} - skipped {easycache.total_steps_skipped}/{total_steps} steps ({speedup:.2f}x speedup).") + easycache.reset() + guider.model_options = orig_model_options + + +class EasyCacheHolder: + def __init__(self, reuse_threshold: float, start_percent: float, end_percent: float, subsample_factor: int, offload_cache_diff: bool, verbose: bool=False, output_channels: int=None): + self.name = "EasyCache" + self.reuse_threshold = reuse_threshold + self.start_percent = start_percent + self.end_percent = end_percent + self.subsample_factor = subsample_factor + self.offload_cache_diff = offload_cache_diff + self.verbose = verbose + # timestep values + self.start_t = 0.0 + self.end_t = 0.0 + # control values + self.relative_transformation_rate: float = None + self.cumulative_change_rate = 0.0 + self.initial_step = True + self.skip_current_step = False + # cache values + self.first_cond_uuid = None + self.x_prev_subsampled: torch.Tensor = None + self.output_prev_subsampled: torch.Tensor = None + self.output_prev_norm: torch.Tensor = None + self.uuid_cache_diffs: dict[UUID, torch.Tensor] = {} + self.uuid_cache_diffs_audio: dict[UUID, torch.Tensor] = {} + self.output_change_rates = [] + self.approx_output_change_rates = [] + self.total_steps_skipped = 0 + # how to deal with mismatched dims + self.allow_mismatch = True + self.cut_from_start = True + self.state_metadata = None + self.output_channels = output_channels + + def is_past_end_timestep(self, timestep: float) -> bool: + return not (timestep[0] > self.end_t).item() + + def should_do_easycache(self, timestep: float) -> bool: + return (timestep[0] <= self.start_t).item() + + def has_x_prev_subsampled(self) -> bool: + return self.x_prev_subsampled is not None + + def has_output_prev_subsampled(self) -> bool: + return self.output_prev_subsampled is not None + + def has_output_prev_norm(self) -> bool: + return self.output_prev_norm is not None + + def has_relative_transformation_rate(self) -> bool: + return self.relative_transformation_rate is not None + + def prepare_timesteps(self, model_sampling): + self.start_t = model_sampling.percent_to_sigma(self.start_percent) + self.end_t = model_sampling.percent_to_sigma(self.end_percent) + return self + + def subsample(self, x: torch.Tensor, uuids: list[UUID], clone: bool = True) -> torch.Tensor: + batch_offset = x.shape[0] // len(uuids) + uuid_idx = uuids.index(self.first_cond_uuid) + if self.subsample_factor > 1: + to_return = x[uuid_idx*batch_offset:(uuid_idx+1)*batch_offset, ..., ::self.subsample_factor, ::self.subsample_factor] + if clone: + return to_return.clone() + return to_return + to_return = x[uuid_idx*batch_offset:(uuid_idx+1)*batch_offset, ...] + if clone: + return to_return.clone() + return to_return + + def can_apply_cache_diff(self, uuids: list[UUID]) -> bool: + return all(uuid in self.uuid_cache_diffs for uuid in uuids) + + def apply_cache_diff(self, x: torch.Tensor, uuids: list[UUID], is_audio: bool = False): + if self.first_cond_uuid in uuids and not is_audio: + self.total_steps_skipped += 1 + cache_diffs = self.uuid_cache_diffs_audio if is_audio else self.uuid_cache_diffs + batch_offset = x.shape[0] // len(uuids) + for i, uuid in enumerate(uuids): + # slice out only what is relevant to this cond + batch_slice = [slice(i*batch_offset,(i+1)*batch_offset)] + # if cached dims don't match x dims, cut off excess and hope for the best (cosmos world2video) + if x.shape[1:] != cache_diffs[uuid].shape[1:]: + if not self.allow_mismatch: + raise ValueError(f"Cached dims {self.uuid_cache_diffs[uuid].shape} don't match x dims {x.shape} - this is no good") + slicing = [] + skip_this_dim = True + for dim_u, dim_x in zip(cache_diffs[uuid].shape, x.shape): + if skip_this_dim: + skip_this_dim = False + continue + if dim_u != dim_x: + if self.cut_from_start: + slicing.append(slice(dim_x-dim_u, None)) + else: + slicing.append(slice(None, dim_u)) + else: + slicing.append(slice(None)) + batch_slice = batch_slice + slicing + x[tuple(batch_slice)] += cache_diffs[uuid].to(x.device) + return x + + def update_cache_diff(self, output: torch.Tensor, x: torch.Tensor, uuids: list[UUID], is_audio: bool = False): + cache_diffs = self.uuid_cache_diffs_audio if is_audio else self.uuid_cache_diffs + # if output dims don't match x dims, cut off excess and hope for the best (cosmos world2video) + if output.shape[1:] != x.shape[1:]: + if not self.allow_mismatch: + raise ValueError(f"Output dims {output.shape} don't match x dims {x.shape} - this is no good") + slicing = [] + skip_dim = True + for dim_o, dim_x in zip(output.shape, x.shape): + if not skip_dim and dim_o != dim_x: + if self.cut_from_start: + slicing.append(slice(dim_x-dim_o, None)) + else: + slicing.append(slice(None, dim_o)) + else: + slicing.append(slice(None)) + skip_dim = False + x = x[tuple(slicing)] + diff = output - x + batch_offset = diff.shape[0] // len(uuids) + for i, uuid in enumerate(uuids): + cache_diffs[uuid] = diff[i*batch_offset:(i+1)*batch_offset, ...] + + def has_first_cond_uuid(self, uuids: list[UUID]) -> bool: + return self.first_cond_uuid in uuids + + def check_metadata(self, x: torch.Tensor) -> bool: + metadata = (x.device, x.dtype, x.shape[1:]) + if self.state_metadata is None: + self.state_metadata = metadata + return True + if metadata == self.state_metadata: + return True + logging.warn(f"{self.name} - Tensor shape, dtype or device changed, resetting state") + self.reset() + return False + + def reset(self): + self.relative_transformation_rate = 0.0 + self.cumulative_change_rate = 0.0 + self.initial_step = True + self.skip_current_step = False + self.output_change_rates = [] + self.first_cond_uuid = None + del self.x_prev_subsampled + self.x_prev_subsampled = None + del self.output_prev_subsampled + self.output_prev_subsampled = None + del self.output_prev_norm + self.output_prev_norm = None + del self.uuid_cache_diffs + self.uuid_cache_diffs = {} + del self.uuid_cache_diffs_audio + self.uuid_cache_diffs_audio = {} + self.total_steps_skipped = 0 + self.state_metadata = None + return self + + def clone(self): + return EasyCacheHolder(self.reuse_threshold, self.start_percent, self.end_percent, self.subsample_factor, self.offload_cache_diff, self.verbose, output_channels=self.output_channels) + + +class EasyCacheNode(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="EasyCache", + display_name="EasyCache", + description="Native EasyCache implementation.", + category="advanced/debug", + is_experimental=True, + inputs=[ + io.Model.Input("model", tooltip="The model to add EasyCache to."), + io.Float.Input("reuse_threshold", min=0.0, default=0.2, max=3.0, step=0.01, tooltip="The threshold for reusing cached steps.", advanced=True), + io.Float.Input("start_percent", min=0.0, default=0.15, max=1.0, step=0.01, tooltip="The relative sampling step to begin use of EasyCache.", advanced=True), + io.Float.Input("end_percent", min=0.0, default=0.95, max=1.0, step=0.01, tooltip="The relative sampling step to end use of EasyCache.", advanced=True), + io.Boolean.Input("verbose", default=False, tooltip="Whether to log verbose information.", advanced=True), + ], + outputs=[ + io.Model.Output(tooltip="The model with EasyCache."), + ], + ) + + @classmethod + def execute(cls, model: io.Model.Type, reuse_threshold: float, start_percent: float, end_percent: float, verbose: bool) -> io.NodeOutput: + model = model.clone() + model.model_options["transformer_options"]["easycache"] = EasyCacheHolder(reuse_threshold, start_percent, end_percent, subsample_factor=8, offload_cache_diff=False, verbose=verbose, output_channels=model.model.latent_format.latent_channels) + model.add_wrapper_with_key(comfy.patcher_extension.WrappersMP.OUTER_SAMPLE, "easycache", easycache_sample_wrapper) + model.add_wrapper_with_key(comfy.patcher_extension.WrappersMP.CALC_COND_BATCH, "easycache", easycache_calc_cond_batch_wrapper) + model.add_wrapper_with_key(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, "easycache", easycache_forward_wrapper) + return io.NodeOutput(model) + + +class LazyCacheHolder: + def __init__(self, reuse_threshold: float, start_percent: float, end_percent: float, subsample_factor: int, offload_cache_diff: bool, verbose: bool=False, output_channels: int=None): + self.name = "LazyCache" + self.reuse_threshold = reuse_threshold + self.start_percent = start_percent + self.end_percent = end_percent + self.subsample_factor = subsample_factor + self.offload_cache_diff = offload_cache_diff + self.verbose = verbose + # timestep values + self.start_t = 0.0 + self.end_t = 0.0 + # control values + self.relative_transformation_rate: float = None + self.cumulative_change_rate = 0.0 + self.initial_step = True + # cache values + self.x_prev_subsampled: torch.Tensor = None + self.output_prev_subsampled: torch.Tensor = None + self.output_prev_norm: torch.Tensor = None + self.cache_diff: torch.Tensor = None + self.output_change_rates = [] + self.approx_output_change_rates = [] + self.total_steps_skipped = 0 + self.state_metadata = None + self.output_channels = output_channels + + def has_cache_diff(self) -> bool: + return self.cache_diff is not None + + def is_past_end_timestep(self, timestep: float) -> bool: + return not (timestep[0] > self.end_t).item() + + def should_do_easycache(self, timestep: float) -> bool: + return (timestep[0] <= self.start_t).item() + + def has_x_prev_subsampled(self) -> bool: + return self.x_prev_subsampled is not None + + def has_output_prev_subsampled(self) -> bool: + return self.output_prev_subsampled is not None + + def has_output_prev_norm(self) -> bool: + return self.output_prev_norm is not None + + def has_relative_transformation_rate(self) -> bool: + return self.relative_transformation_rate is not None + + def prepare_timesteps(self, model_sampling): + self.start_t = model_sampling.percent_to_sigma(self.start_percent) + self.end_t = model_sampling.percent_to_sigma(self.end_percent) + return self + + def subsample(self, x: torch.Tensor, clone: bool = True) -> torch.Tensor: + if self.subsample_factor > 1: + to_return = x[..., ::self.subsample_factor, ::self.subsample_factor] + if clone: + return to_return.clone() + return to_return + if clone: + return x.clone() + return x + + def apply_cache_diff(self, x: torch.Tensor): + self.total_steps_skipped += 1 + return x + self.cache_diff.to(x.device) + + def update_cache_diff(self, output: torch.Tensor, x: torch.Tensor): + self.cache_diff = output - x + + def check_metadata(self, x: torch.Tensor) -> bool: + metadata = (x.device, x.dtype, x.shape) + if self.state_metadata is None: + self.state_metadata = metadata + return True + if metadata == self.state_metadata: + return True + logging.warn(f"{self.name} - Tensor shape, dtype or device changed, resetting state") + self.reset() + return False + + def reset(self): + self.relative_transformation_rate = 0.0 + self.cumulative_change_rate = 0.0 + self.initial_step = True + self.output_change_rates = [] + self.approx_output_change_rates = [] + del self.cache_diff + self.cache_diff = None + del self.x_prev_subsampled + self.x_prev_subsampled = None + del self.output_prev_subsampled + self.output_prev_subsampled = None + del self.output_prev_norm + self.output_prev_norm = None + self.total_steps_skipped = 0 + self.state_metadata = None + return self + + def clone(self): + return LazyCacheHolder(self.reuse_threshold, self.start_percent, self.end_percent, self.subsample_factor, self.offload_cache_diff, self.verbose, output_channels=self.output_channels) + +class LazyCacheNode(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="LazyCache", + display_name="LazyCache", + description="A homebrew version of EasyCache - even 'easier' version of EasyCache to implement. Overall works worse than EasyCache, but better in some rare cases AND universal compatibility with everything in ComfyUI.", + category="advanced/debug", + is_experimental=True, + inputs=[ + io.Model.Input("model", tooltip="The model to add LazyCache to."), + io.Float.Input("reuse_threshold", min=0.0, default=0.2, max=3.0, step=0.01, tooltip="The threshold for reusing cached steps.", advanced=True), + io.Float.Input("start_percent", min=0.0, default=0.15, max=1.0, step=0.01, tooltip="The relative sampling step to begin use of LazyCache.", advanced=True), + io.Float.Input("end_percent", min=0.0, default=0.95, max=1.0, step=0.01, tooltip="The relative sampling step to end use of LazyCache.", advanced=True), + io.Boolean.Input("verbose", default=False, tooltip="Whether to log verbose information.", advanced=True), + ], + outputs=[ + io.Model.Output(tooltip="The model with LazyCache."), + ], + ) + + @classmethod + def execute(cls, model: io.Model.Type, reuse_threshold: float, start_percent: float, end_percent: float, verbose: bool) -> io.NodeOutput: + model = model.clone() + model.model_options["transformer_options"]["easycache"] = LazyCacheHolder(reuse_threshold, start_percent, end_percent, subsample_factor=8, offload_cache_diff=False, verbose=verbose, output_channels=model.model.latent_format.latent_channels) + model.add_wrapper_with_key(comfy.patcher_extension.WrappersMP.OUTER_SAMPLE, "lazycache", easycache_sample_wrapper) + model.add_wrapper_with_key(comfy.patcher_extension.WrappersMP.PREDICT_NOISE, "lazycache", lazycache_predict_noise_wrapper) + return io.NodeOutput(model) + + +class EasyCacheExtension(ComfyExtension): + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + EasyCacheNode, + LazyCacheNode, + ] + +def comfy_entrypoint(): + return EasyCacheExtension() diff --git a/ComfyUI/comfy_extras/nodes_edit_model.py b/ComfyUI/comfy_extras/nodes_edit_model.py new file mode 100644 index 0000000000000000000000000000000000000000..d0d20ae7a404aa91b6cf9bb823194a7ed2de0699 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_edit_model.py @@ -0,0 +1,39 @@ +import node_helpers +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io + + +class ReferenceLatent(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ReferenceLatent", + display_name="Set Reference Latent", + category="model/conditioning", + description="This node sets the guiding latent for an edit model. If the model supports it you can chain multiple to set multiple reference images.", + inputs=[ + io.Conditioning.Input("conditioning"), + io.Latent.Input("latent", optional=True), + ], + outputs=[ + io.Conditioning.Output(), + ] + ) + + @classmethod + def execute(cls, conditioning, latent=None) -> io.NodeOutput: + if latent is not None: + conditioning = node_helpers.conditioning_set_values(conditioning, {"reference_latents": [latent["samples"]]}, append=True) + return io.NodeOutput(conditioning) + + +class EditModelExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + ReferenceLatent, + ] + + +def comfy_entrypoint() -> EditModelExtension: + return EditModelExtension() diff --git a/ComfyUI/comfy_extras/nodes_eps.py b/ComfyUI/comfy_extras/nodes_eps.py new file mode 100644 index 0000000000000000000000000000000000000000..8c397f132a83907c5bd78a9d5bdbcac26aab01c0 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_eps.py @@ -0,0 +1,172 @@ +import torch +from typing_extensions import override + +from comfy.k_diffusion.sampling import sigma_to_half_log_snr +from comfy_api.latest import ComfyExtension, io + + +class EpsilonScaling(io.ComfyNode): + """ + Implements the Epsilon Scaling method from 'Elucidating the Exposure Bias in Diffusion Models' + (https://arxiv.org/abs/2308.15321v6). + + This method mitigates exposure bias by scaling the predicted noise during sampling, + which can significantly improve sample quality. This implementation uses the "uniform schedule" + recommended by the paper for its practicality and effectiveness. + """ + @classmethod + def define_schema(cls): + return io.Schema( + node_id="Epsilon Scaling", + category="model/patch/unet", + inputs=[ + io.Model.Input("model"), + io.Float.Input( + "scaling_factor", + default=1.005, + min=0.5, + max=1.5, + step=0.001, + display_mode=io.NumberDisplay.number, + advanced=True, + ), + ], + outputs=[ + io.Model.Output(), + ], + ) + + @classmethod + def execute(cls, model, scaling_factor) -> io.NodeOutput: + # Prevent division by zero, though the UI's min value should prevent this. + if scaling_factor == 0: + scaling_factor = 1e-9 + + def epsilon_scaling_function(args): + """ + This function is applied after the CFG guidance has been calculated. + It recalculates the denoised latent by scaling the predicted noise. + """ + denoised = args["denoised"] + x = args["input"] + + noise_pred = x - denoised + + scaled_noise_pred = noise_pred / scaling_factor + + new_denoised = x - scaled_noise_pred + + return new_denoised + + # Clone the model patcher to avoid modifying the original model in place + model_clone = model.clone() + + model_clone.set_model_sampler_post_cfg_function(epsilon_scaling_function) + + return io.NodeOutput(model_clone) + + +def compute_tsr_rescaling_factor( + snr: torch.Tensor, tsr_k: float, tsr_variance: float +) -> torch.Tensor: + """Compute the rescaling score ratio in Temporal Score Rescaling. + + See equation (6) in https://arxiv.org/pdf/2510.01184v1. + """ + posinf_mask = torch.isposinf(snr) + rescaling_factor = (snr * tsr_variance + 1) / (snr * tsr_variance / tsr_k + 1) + return torch.where(posinf_mask, tsr_k, rescaling_factor) # when snr → inf, r = tsr_k + + +class TemporalScoreRescaling(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="TemporalScoreRescaling", + display_name="TSR - Temporal Score Rescaling", + category="model/patch/unet", + inputs=[ + io.Model.Input("model"), + io.Float.Input( + "tsr_k", + tooltip=( + "Controls the rescaling strength.\n" + "Lower k produces more detailed results; higher k produces smoother results in image generation. Setting k = 1 disables rescaling." + ), + default=0.95, + min=0.01, + max=100.0, + step=0.001, + display_mode=io.NumberDisplay.number, + advanced=True, + ), + io.Float.Input( + "tsr_sigma", + tooltip=( + "Controls how early rescaling takes effect.\n" + "Larger values take effect earlier." + ), + default=1.0, + min=0.01, + max=100.0, + step=0.001, + display_mode=io.NumberDisplay.number, + advanced=True, + ), + ], + outputs=[ + io.Model.Output( + display_name="patched_model", + ), + ], + description=( + "[Post-CFG Function]\n" + "TSR - Temporal Score Rescaling (2510.01184)\n\n" + "Rescaling the model's score or noise to steer the sampling diversity.\n" + ), + ) + + @classmethod + def execute(cls, model, tsr_k, tsr_sigma) -> io.NodeOutput: + tsr_variance = tsr_sigma**2 + + def temporal_score_rescaling(args): + denoised = args["denoised"] + x = args["input"] + sigma = args["sigma"] + curr_model = args["model"] + + # No rescaling (r = 1) or no noise + if tsr_k == 1 or sigma == 0: + return denoised + + model_sampling = curr_model.current_patcher.get_model_object("model_sampling") + half_log_snr = sigma_to_half_log_snr(sigma, model_sampling) + snr = (2 * half_log_snr).exp() + + # No rescaling needed (r = 1) + if snr == 0: + return denoised + + rescaling_r = compute_tsr_rescaling_factor(snr, tsr_k, tsr_variance) + + # Derived from scaled_denoised = (x - r * sigma * noise) / alpha + alpha = sigma * half_log_snr.exp() + return torch.lerp(x / alpha, denoised, rescaling_r) + + m = model.clone() + m.set_model_sampler_post_cfg_function(temporal_score_rescaling) + return io.NodeOutput(m) + + +class EpsilonScalingExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + EpsilonScaling, + TemporalScoreRescaling, + ] + + +async def comfy_entrypoint() -> EpsilonScalingExtension: + return EpsilonScalingExtension() diff --git a/ComfyUI/comfy_extras/nodes_flux.py b/ComfyUI/comfy_extras/nodes_flux.py new file mode 100644 index 0000000000000000000000000000000000000000..e9986c9e7227c4217029961addaed447b0a66b2c --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_flux.py @@ -0,0 +1,319 @@ +import node_helpers +import comfy.utils +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io +import comfy.model_management +import torch +import math +import nodes +import comfy.ldm.flux.math + +class CLIPTextEncodeFlux(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="CLIPTextEncodeFlux", + category="model/conditioning/flux", + inputs=[ + io.Clip.Input("clip"), + io.String.Input("clip_l", multiline=True, dynamic_prompts=True), + io.String.Input("t5xxl", multiline=True, dynamic_prompts=True), + io.Float.Input("guidance", default=3.5, min=0.0, max=100.0, step=0.1), + ], + outputs=[ + io.Conditioning.Output(), + ], + ) + + @classmethod + def execute(cls, clip, clip_l, t5xxl, guidance) -> io.NodeOutput: + tokens = clip.tokenize(clip_l) + tokens["t5xxl"] = clip.tokenize(t5xxl)["t5xxl"] + + return io.NodeOutput(clip.encode_from_tokens_scheduled(tokens, add_dict={"guidance": guidance})) + + encode = execute # TODO: remove + +class EmptyFlux2LatentImage(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="EmptyFlux2LatentImage", + display_name="Empty Flux 2 Latent", + category="model/latent/flux", + inputs=[ + io.Int.Input("width", default=1024, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("height", default=1024, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("batch_size", default=1, min=1, max=4096), + ], + outputs=[ + io.Latent.Output(), + ], + ) + + @classmethod + def execute(cls, width, height, batch_size=1) -> io.NodeOutput: + latent = torch.zeros([batch_size, 128, height // 16, width // 16], device=comfy.model_management.intermediate_device()) + return io.NodeOutput({"samples": latent}) + +class FluxGuidance(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="FluxGuidance", + category="model/conditioning/flux", + inputs=[ + io.Conditioning.Input("conditioning"), + io.Float.Input("guidance", default=3.5, min=0.0, max=100.0, step=0.1), + ], + outputs=[ + io.Conditioning.Output(), + ], + ) + + @classmethod + def execute(cls, conditioning, guidance) -> io.NodeOutput: + c = node_helpers.conditioning_set_values(conditioning, {"guidance": guidance}) + return io.NodeOutput(c) + + append = execute # TODO: remove + + +class FluxDisableGuidance(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="FluxDisableGuidance", + category="model/conditioning/flux", + description="This node completely disables the guidance embed on Flux and Flux like models", + inputs=[ + io.Conditioning.Input("conditioning"), + ], + outputs=[ + io.Conditioning.Output(), + ], + ) + + @classmethod + def execute(cls, conditioning) -> io.NodeOutput: + c = node_helpers.conditioning_set_values(conditioning, {"guidance": None}) + return io.NodeOutput(c) + + append = execute # TODO: remove + + +PREFERRED_KONTEXT_RESOLUTIONS = [ + (672, 1568), + (688, 1504), + (720, 1456), + (752, 1392), + (800, 1328), + (832, 1248), + (880, 1184), + (944, 1104), + (1024, 1024), + (1104, 944), + (1184, 880), + (1248, 832), + (1328, 800), + (1392, 752), + (1456, 720), + (1504, 688), + (1568, 672), +] + + +class FluxKontextImageScale(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="FluxKontextImageScale", + category="model/conditioning/flux", + description="This node resizes the image to one that is more optimal for flux kontext.", + inputs=[ + io.Image.Input("image"), + ], + outputs=[ + io.Image.Output(), + ], + ) + + @classmethod + def execute(cls, image) -> io.NodeOutput: + width = image.shape[2] + height = image.shape[1] + aspect_ratio = width / height + _, width, height = min((abs(aspect_ratio - w / h), w, h) for w, h in PREFERRED_KONTEXT_RESOLUTIONS) + image = comfy.utils.common_upscale(image.movedim(-1, 1), width, height, "lanczos", "center").movedim(1, -1) + return io.NodeOutput(image) + + scale = execute # TODO: remove + + +class FluxKontextMultiReferenceLatentMethod(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="FluxKontextMultiReferenceLatentMethod", + display_name="Edit Model Reference Method", + category="model/conditioning/flux", + inputs=[ + io.Conditioning.Input("conditioning"), + io.Combo.Input( + "reference_latents_method", + options=["offset", "index", "uxo/uno", "index_timestep_zero"], + advanced=True, + ), + ], + outputs=[ + io.Conditioning.Output(), + ], + is_experimental=True, + ) + + @classmethod + def execute(cls, conditioning, reference_latents_method) -> io.NodeOutput: + if "uxo" in reference_latents_method or "uso" in reference_latents_method: + reference_latents_method = "uxo" + c = node_helpers.conditioning_set_values(conditioning, {"reference_latents_method": reference_latents_method}) + return io.NodeOutput(c) + + append = execute # TODO: remove + + +def generalized_time_snr_shift(t, mu: float, sigma: float): + return math.exp(mu) / (math.exp(mu) + (1 / t - 1) ** sigma) + + +def compute_empirical_mu(image_seq_len: int, num_steps: int) -> float: + a1, b1 = 8.73809524e-05, 1.89833333 + a2, b2 = 0.00016927, 0.45666666 + + if image_seq_len > 4300: + mu = a2 * image_seq_len + b2 + return float(mu) + + m_200 = a2 * image_seq_len + b2 + m_10 = a1 * image_seq_len + b1 + + a = (m_200 - m_10) / 190.0 + b = m_200 - 200.0 * a + mu = a * num_steps + b + + return float(mu) + + +def get_schedule(num_steps: int, image_seq_len: int) -> list[float]: + mu = compute_empirical_mu(image_seq_len, num_steps) + timesteps = torch.linspace(1, 0, num_steps + 1) + timesteps = generalized_time_snr_shift(timesteps, mu, 1.0) + return timesteps + + +class Flux2Scheduler(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="Flux2Scheduler", + category="model/sampling/schedulers", + inputs=[ + io.Int.Input("steps", default=20, min=1, max=4096), + io.Int.Input("width", default=1024, min=16, max=nodes.MAX_RESOLUTION, step=1), + io.Int.Input("height", default=1024, min=16, max=nodes.MAX_RESOLUTION, step=1), + ], + outputs=[ + io.Sigmas.Output(), + ], + ) + + @classmethod + def execute(cls, steps, width, height) -> io.NodeOutput: + seq_len = (width * height / (16 * 16)) + sigmas = get_schedule(steps, round(seq_len)) + return io.NodeOutput(sigmas) + +class KV_Attn_Input: + def __init__(self): + self.cache = {} + + def __call__(self, q, k, v, extra_options, **kwargs): + reference_image_num_tokens = extra_options.get("reference_image_num_tokens", []) + if len(reference_image_num_tokens) == 0: + return {} + + ref_toks = sum(reference_image_num_tokens) + cache_key = "{}_{}".format(extra_options["block_type"], extra_options["block_index"]) + if cache_key in self.cache: + kk, vv = self.cache[cache_key] + + # Fix batch size changing. + kk = comfy.utils.repeat_to_batch_size(kk, k.shape[0]) + vv = comfy.utils.repeat_to_batch_size(vv, v.shape[0]) + + self.set_cache = False + return {"q": q, "k": torch.cat((k, kk), dim=2), "v": torch.cat((v, vv), dim=2)} + + self.cache[cache_key] = (k[:, :, -ref_toks:].clone(), v[:, :, -ref_toks:].clone()) + self.set_cache = True + return {"q": q, "k": k, "v": v} + + def cleanup(self): + self.cache = {} + + +class FluxKVCache(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="FluxKVCache", + display_name="Flux KV Cache", + description="Enables KV Cache optimization for reference images on Flux family models.", + category="experimental", + is_experimental=True, + inputs=[ + io.Model.Input("model", tooltip="The model to use KV Cache on."), + ], + outputs=[ + io.Model.Output(tooltip="The patched model with KV Cache enabled."), + ], + ) + + @classmethod + def execute(cls, model: io.Model.Type) -> io.NodeOutput: + m = model.clone() + input_patch_obj = KV_Attn_Input() + + def model_input_patch(inputs): + if len(input_patch_obj.cache) > 0: + ref_image_tokens = sum(inputs["transformer_options"].get("reference_image_num_tokens", [])) + if ref_image_tokens > 0: + img = inputs["img"] + inputs["img"] = img[:, :-ref_image_tokens] + return inputs + + m.set_model_attn1_patch(input_patch_obj) + m.set_model_post_input_patch(model_input_patch) + if hasattr(model.model.diffusion_model, "params"): + m.add_object_patch("diffusion_model.params.default_ref_method", "index_timestep_zero") + else: + m.add_object_patch("diffusion_model.default_ref_method", "index_timestep_zero") + + return io.NodeOutput(m) + +class FluxExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + CLIPTextEncodeFlux, + FluxGuidance, + FluxDisableGuidance, + FluxKontextImageScale, + FluxKontextMultiReferenceLatentMethod, + EmptyFlux2LatentImage, + Flux2Scheduler, + FluxKVCache, + ] + + +async def comfy_entrypoint() -> FluxExtension: + return FluxExtension() diff --git a/ComfyUI/comfy_extras/nodes_frame_interpolation.py b/ComfyUI/comfy_extras/nodes_frame_interpolation.py new file mode 100644 index 0000000000000000000000000000000000000000..44708e5ec4bce04832a548d63d6ceef69ce98ecd --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_frame_interpolation.py @@ -0,0 +1,208 @@ +import torch +from tqdm import tqdm +from typing_extensions import override + +import comfy.model_patcher +import comfy.utils +import folder_paths +from comfy import model_management +from comfy_extras.frame_interpolation_models.ifnet import IFNet, detect_rife_config +from comfy_extras.frame_interpolation_models.film_net import FILMNet +from comfy_api.latest import ComfyExtension, io + +FrameInterpolationModel = io.Custom("INTERP_MODEL") + + +class FrameInterpolationModelLoader(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="FrameInterpolationModelLoader", + display_name="Load Frame Interpolation Model", + category="model/loaders", + inputs=[ + io.Combo.Input("model_name", options=folder_paths.get_filename_list("frame_interpolation"), + tooltip="Select a frame interpolation model to load. Models must be placed in the 'frame_interpolation' folder."), + ], + outputs=[ + FrameInterpolationModel.Output(), + ], + ) + + @classmethod + def execute(cls, model_name) -> io.NodeOutput: + model_path = folder_paths.get_full_path_or_raise("frame_interpolation", model_name) + sd = comfy.utils.load_torch_file(model_path, safe_load=True) + + model = cls._detect_and_load(sd) + dtype = torch.float16 if model_management.should_use_fp16(model_management.get_torch_device()) else torch.float32 + model.eval().to(dtype) + patcher = comfy.model_patcher.CoreModelPatcher( + model, + load_device=model_management.get_torch_device(), + offload_device=model_management.unet_offload_device(), + ) + return io.NodeOutput(patcher) + + @classmethod + def _detect_and_load(cls, sd): + # Try FILM + if "extract.extract_sublevels.convs.0.0.conv.weight" in sd: + model = FILMNet() + model.load_state_dict(sd) + return model + + # Try RIFE (needs key remapping for raw checkpoints) + sd = comfy.utils.state_dict_prefix_replace(sd, {"module.": "", "flownet.": ""}) + key_map = {} + for k in sd: + for i in range(5): + if k.startswith(f"block{i}."): + key_map[k] = f"blocks.{i}.{k[len(f'block{i}.'):]}" + if key_map: + sd = {key_map.get(k, k): v for k, v in sd.items()} + sd = {k: v for k, v in sd.items() if not k.startswith(("teacher.", "caltime."))} + + try: + head_ch, channels = detect_rife_config(sd) + except (KeyError, ValueError): + raise ValueError("Unrecognized frame interpolation model format") + model = IFNet(head_ch=head_ch, channels=channels) + model.load_state_dict(sd) + return model + + +class FrameInterpolate(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="FrameInterpolate", + display_name="Run Frame Interpolation Model", + category="video", + search_aliases=["rife", "film", "frame interpolation", "slow motion", "interpolate frames", "vfi"], + inputs=[ + FrameInterpolationModel.Input("interp_model"), + io.Image.Input("images"), + io.Int.Input("multiplier", default=2, min=2, max=16), + ], + outputs=[ + io.Image.Output(), + ], + ) + + @classmethod + def execute(cls, interp_model, images, multiplier) -> io.NodeOutput: + offload_device = model_management.intermediate_device() + + num_frames = images.shape[0] + if num_frames < 2 or multiplier < 2: + return io.NodeOutput(images) + + device = interp_model.load_device + dtype = interp_model.model_dtype() + inference_model = interp_model.model + activation_mem = inference_model.memory_used_forward(images.shape, dtype) + model_management.load_models_gpu([interp_model], memory_required=activation_mem) + align = getattr(inference_model, "pad_align", 1) + H, W = images.shape[1], images.shape[2] + + # Prepare a single padded frame on device for determining output dimensions + def prepare_frame(idx): + frame = images[idx:idx + 1].movedim(-1, 1).to(dtype=dtype, device=device) + if align > 1: + from comfy.ldm.common_dit import pad_to_patch_size + frame = pad_to_patch_size(frame, (align, align), padding_mode="reflect") + return frame + + # Count total interpolation passes for progress bar + total_pairs = num_frames - 1 + num_interp = multiplier - 1 + total_steps = total_pairs * num_interp + pbar = comfy.utils.ProgressBar(total_steps) + tqdm_bar = tqdm(total=total_steps, desc="Frame interpolation") + + batch = num_interp # reduced on OOM and persists across pairs (same resolution = same limit) + t_values = [t / multiplier for t in range(1, multiplier)] + + out_dtype = model_management.intermediate_dtype() + total_out_frames = total_pairs * multiplier + 1 + result = torch.empty((total_out_frames, 3, H, W), dtype=out_dtype, device=offload_device) + result[0] = images[0].movedim(-1, 0).to(out_dtype) + out_idx = 1 + + # Pre-compute timestep tensor on device (padded dimensions needed) + sample = prepare_frame(0) + pH, pW = sample.shape[2], sample.shape[3] + ts_full = torch.tensor(t_values, device=device, dtype=dtype).reshape(num_interp, 1, 1, 1) + ts_full = ts_full.expand(-1, 1, pH, pW) + del sample + + multi_fn = getattr(inference_model, "forward_multi_timestep", None) + feat_cache = {} + prev_frame = None + + try: + for i in range(total_pairs): + img0_single = prev_frame if prev_frame is not None else prepare_frame(i) + img1_single = prepare_frame(i + 1) + prev_frame = img1_single + + # Cache features: img1 of pair N becomes img0 of pair N+1 + feat_cache["img0"] = feat_cache.pop("next") if "next" in feat_cache else inference_model.extract_features(img0_single) + feat_cache["img1"] = inference_model.extract_features(img1_single) + feat_cache["next"] = feat_cache["img1"] + + used_multi = False + if multi_fn is not None: + # Models with timestep-independent flow can compute it once for all timesteps + try: + mids = multi_fn(img0_single, img1_single, t_values, cache=feat_cache) + result[out_idx:out_idx + num_interp] = mids[:, :, :H, :W].to(out_dtype) + out_idx += num_interp + pbar.update(num_interp) + tqdm_bar.update(num_interp) + used_multi = True + except model_management.OOM_EXCEPTION: + model_management.soft_empty_cache() + multi_fn = None # fall through to single-timestep path + + if not used_multi: + j = 0 + while j < num_interp: + b = min(batch, num_interp - j) + try: + img0 = img0_single.expand(b, -1, -1, -1) + img1 = img1_single.expand(b, -1, -1, -1) + mids = inference_model(img0, img1, timestep=ts_full[j:j + b], cache=feat_cache) + result[out_idx:out_idx + b] = mids[:, :, :H, :W].to(out_dtype) + out_idx += b + pbar.update(b) + tqdm_bar.update(b) + j += b + except model_management.OOM_EXCEPTION: + if batch <= 1: + raise + batch = max(1, batch // 2) + model_management.soft_empty_cache() + + result[out_idx] = images[i + 1].movedim(-1, 0).to(out_dtype) + out_idx += 1 + finally: + tqdm_bar.close() + + # BCHW -> BHWC + result = result.movedim(1, -1).clamp_(0.0, 1.0) + return io.NodeOutput(result) + + +class FrameInterpolationExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + FrameInterpolationModelLoader, + FrameInterpolate, + ] + + +async def comfy_entrypoint() -> FrameInterpolationExtension: + return FrameInterpolationExtension() diff --git a/ComfyUI/comfy_extras/nodes_freelunch.py b/ComfyUI/comfy_extras/nodes_freelunch.py new file mode 100644 index 0000000000000000000000000000000000000000..ccbd1fd905102fbe408d3bf764b653b13f3b431b --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_freelunch.py @@ -0,0 +1,138 @@ +#code originally taken from: https://github.com/ChenyangSi/FreeU (under MIT License) + +import torch +import logging +from typing_extensions import override +from comfy_api.latest import ComfyExtension, IO + +def Fourier_filter(x, threshold, scale): + # FFT + x_freq = torch.fft.fftn(x.float(), dim=(-2, -1)) + x_freq = torch.fft.fftshift(x_freq, dim=(-2, -1)) + + B, C, H, W = x_freq.shape + mask = torch.ones((B, C, H, W), device=x.device) + + crow, ccol = H // 2, W //2 + mask[..., crow - threshold:crow + threshold, ccol - threshold:ccol + threshold] = scale + x_freq = x_freq * mask + + # IFFT + x_freq = torch.fft.ifftshift(x_freq, dim=(-2, -1)) + x_filtered = torch.fft.ifftn(x_freq, dim=(-2, -1)).real + + return x_filtered.to(x.dtype) + + +class FreeU(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="FreeU", + category="model/patch/unet", + inputs=[ + IO.Model.Input("model"), + IO.Float.Input("b1", default=1.1, min=0.0, max=10.0, step=0.01, advanced=True), + IO.Float.Input("b2", default=1.2, min=0.0, max=10.0, step=0.01, advanced=True), + IO.Float.Input("s1", default=0.9, min=0.0, max=10.0, step=0.01, advanced=True), + IO.Float.Input("s2", default=0.2, min=0.0, max=10.0, step=0.01, advanced=True), + ], + outputs=[ + IO.Model.Output(), + ], + ) + + @classmethod + def execute(cls, model, b1, b2, s1, s2) -> IO.NodeOutput: + model_channels = model.model.model_config.unet_config["model_channels"] + scale_dict = {model_channels * 4: (b1, s1), model_channels * 2: (b2, s2)} + on_cpu_devices = {} + + def output_block_patch(h, hsp, transformer_options): + scale = scale_dict.get(int(h.shape[1]), None) + if scale is not None: + h[:,:h.shape[1] // 2] = h[:,:h.shape[1] // 2] * scale[0] + if hsp.device not in on_cpu_devices: + try: + hsp = Fourier_filter(hsp, threshold=1, scale=scale[1]) + except: + logging.warning("Device {} does not support the torch.fft functions used in the FreeU node, switching to CPU.".format(hsp.device)) + on_cpu_devices[hsp.device] = True + hsp = Fourier_filter(hsp.cpu(), threshold=1, scale=scale[1]).to(hsp.device) + else: + hsp = Fourier_filter(hsp.cpu(), threshold=1, scale=scale[1]).to(hsp.device) + + return h, hsp + + m = model.clone() + m.set_model_output_block_patch(output_block_patch) + return IO.NodeOutput(m) + + patch = execute # TODO: remove + + +class FreeU_V2(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="FreeU_V2", + category="model/patch/unet", + inputs=[ + IO.Model.Input("model"), + IO.Float.Input("b1", default=1.3, min=0.0, max=10.0, step=0.01, advanced=True), + IO.Float.Input("b2", default=1.4, min=0.0, max=10.0, step=0.01, advanced=True), + IO.Float.Input("s1", default=0.9, min=0.0, max=10.0, step=0.01, advanced=True), + IO.Float.Input("s2", default=0.2, min=0.0, max=10.0, step=0.01, advanced=True), + ], + outputs=[ + IO.Model.Output(), + ], + ) + + @classmethod + def execute(cls, model, b1, b2, s1, s2) -> IO.NodeOutput: + model_channels = model.model.model_config.unet_config["model_channels"] + scale_dict = {model_channels * 4: (b1, s1), model_channels * 2: (b2, s2)} + on_cpu_devices = {} + + def output_block_patch(h, hsp, transformer_options): + scale = scale_dict.get(int(h.shape[1]), None) + if scale is not None: + hidden_mean = h.mean(1).unsqueeze(1) + B = hidden_mean.shape[0] + hidden_max, _ = torch.max(hidden_mean.view(B, -1), dim=-1, keepdim=True) + hidden_min, _ = torch.min(hidden_mean.view(B, -1), dim=-1, keepdim=True) + hidden_mean = (hidden_mean - hidden_min.unsqueeze(2).unsqueeze(3)) / (hidden_max - hidden_min).unsqueeze(2).unsqueeze(3) + + h[:,:h.shape[1] // 2] = h[:,:h.shape[1] // 2] * ((scale[0] - 1 ) * hidden_mean + 1) + + if hsp.device not in on_cpu_devices: + try: + hsp = Fourier_filter(hsp, threshold=1, scale=scale[1]) + except: + logging.warning("Device {} does not support the torch.fft functions used in the FreeU node, switching to CPU.".format(hsp.device)) + on_cpu_devices[hsp.device] = True + hsp = Fourier_filter(hsp.cpu(), threshold=1, scale=scale[1]).to(hsp.device) + else: + hsp = Fourier_filter(hsp.cpu(), threshold=1, scale=scale[1]).to(hsp.device) + + return h, hsp + + m = model.clone() + m.set_model_output_block_patch(output_block_patch) + return IO.NodeOutput(m) + + patch = execute # TODO: remove + + +class FreelunchExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[IO.ComfyNode]]: + return [ + FreeU, + FreeU_V2, + ] + + +async def comfy_entrypoint() -> FreelunchExtension: + return FreelunchExtension() diff --git a/ComfyUI/comfy_extras/nodes_fresca.py b/ComfyUI/comfy_extras/nodes_fresca.py new file mode 100644 index 0000000000000000000000000000000000000000..a7d181bdf6029c46695ca59a72940315cdedd8c4 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_fresca.py @@ -0,0 +1,115 @@ +# Code based on https://github.com/WikiChao/FreSca (MIT License) +import torch +import torch.fft as fft +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io + + +def Fourier_filter(x, scale_low=1.0, scale_high=1.5, freq_cutoff=20): + """ + Apply frequency-dependent scaling to an image tensor using Fourier transforms. + + Parameters: + x: Input tensor of shape (..., H, W) + scale_low: Scaling factor for low-frequency components (default: 1.0) + scale_high: Scaling factor for high-frequency components (default: 1.5) + freq_cutoff: Number of frequency indices around center to consider as low-frequency (default: 20) + + Returns: + x_filtered: Filtered version of x in spatial domain with frequency-specific scaling applied. + """ + # Preserve input dtype and device + dtype, device = x.dtype, x.device + + # Convert to float32 for FFT computations + x = x.to(torch.float32) + + # 1) Apply FFT and shift low frequencies to center + x_freq = fft.fftn(x, dim=(-2, -1)) + x_freq = fft.fftshift(x_freq, dim=(-2, -1)) + + # Initialize mask with high-frequency scaling factor + mask = torch.ones(x_freq.shape, device=device) * scale_high + m = mask + for d in range(2): + dim = len(x_freq.shape) - 2 + d + cc = x_freq.shape[dim] // 2 + f_c = min(freq_cutoff, cc) + m = m.narrow(dim, cc - f_c, f_c * 2) + + # Apply low-frequency scaling factor to center region + m[:] = scale_low + + # 3) Apply frequency-specific scaling + x_freq = x_freq * mask + + # 4) Convert back to spatial domain + x_freq = fft.ifftshift(x_freq, dim=(-2, -1)) + x_filtered = fft.ifftn(x_freq, dim=(-2, -1)).real + + # 5) Restore original dtype + x_filtered = x_filtered.to(dtype) + + return x_filtered + + +class FreSca(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="FreSca", + search_aliases=["frequency guidance"], + display_name="FreSca", + category="experimental", + description="Applies frequency-dependent scaling to the guidance", + inputs=[ + io.Model.Input("model"), + io.Float.Input("scale_low", default=1.0, min=0, max=10, step=0.01, + tooltip="Scaling factor for low-frequency components", advanced=True), + io.Float.Input("scale_high", default=1.25, min=0, max=10, step=0.01, + tooltip="Scaling factor for high-frequency components", advanced=True), + io.Int.Input("freq_cutoff", default=20, min=1, max=10000, step=1, + tooltip="Number of frequency indices around center to consider as low-frequency", advanced=True), + ], + outputs=[ + io.Model.Output(), + ], + is_experimental=True, + ) + + @classmethod + def execute(cls, model, scale_low, scale_high, freq_cutoff): + def custom_cfg_function(args): + conds_out = args["conds_out"] + if len(conds_out) <= 1 or None in args["conds"][:2]: + return conds_out + cond = conds_out[0] + uncond = conds_out[1] + + guidance = cond - uncond + filtered_guidance = Fourier_filter( + guidance, + scale_low=scale_low, + scale_high=scale_high, + freq_cutoff=freq_cutoff, + ) + filtered_cond = filtered_guidance + uncond + + return [filtered_cond, uncond] + conds_out[2:] + + m = model.clone() + m.set_model_sampler_pre_cfg_function(custom_cfg_function) + + return io.NodeOutput(m) + + +class FreScaExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + FreSca, + ] + + +async def comfy_entrypoint() -> FreScaExtension: + return FreScaExtension() diff --git a/ComfyUI/comfy_extras/nodes_gaussian_splat.py b/ComfyUI/comfy_extras/nodes_gaussian_splat.py new file mode 100644 index 0000000000000000000000000000000000000000..a53317b4e56dda33f95cfca83f3d0ba03d2a8be7 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_gaussian_splat.py @@ -0,0 +1,1673 @@ +# Generic utility nodes for the SPLAT type (3D gaussian splats) + +import gzip +import logging +import math +import struct +from io import BytesIO + +import numpy as np +import torch +from typing_extensions import override +from scipy.ndimage import map_coordinates, minimum as _ndi_minimum, maximum as _ndi_maximum +from scipy.sparse import coo_matrix +from scipy.sparse.csgraph import connected_components + +import comfy.model_management +import comfy.utils +from comfy_api.latest import ComfyExtension, IO, Types +from comfy_extras.nodes_save_3d import pack_variable_mesh_batch +from server import PromptServer + +_C0 = 0.28209479177387814 # SH band-0 constant: DC coefficient -> base RGB + + +def _srgb_to_linear(c): + return torch.where(c <= 0.04045, c / 12.92, ((c.clamp_min(0) + 0.055) / 1.055) ** 2.4) + + +def _linear_to_srgb(c): + return torch.where(c <= 0.0031308, c * 12.92, 1.055 * c.clamp_min(0) ** (1 / 2.4) - 0.055) + + +def _real_len(g: Types.SPLAT, i: int) -> int: + # Real splat count of batch item i (honors variable-length `counts`). + return int(g.counts[i].item()) if g.counts is not None else g.positions.shape[1] + + +def _hex_to_rgb(h: str) -> tuple[float, float, float]: + # "#RRGGBB" -> (r,g,b) in [0,1]; falls back to black. + h = h.lstrip("#") + if len(h) != 6: + return (0.0, 0.0, 0.0) + return tuple(int(h[i:i + 2], 16) / 255.0 for i in (0, 2, 4)) + + +def _quantile(x, q): + # torch.quantile errors above 2**24 elements; stride-subsample large inputs for the estimate. + lim = 1 << 24 + if x.numel() > lim: + x = x[:: x.numel() // lim + 1] + return torch.quantile(x, q) + + +def _gaussian_ply_bytes(positions, scales, rotations, opacities, sh) -> bytes: + """Serialize render-ready gaussian tensors as a binary 3DGS .ply. + + positions (N,3) world; scales (N,3) linear; rotations (N,4) quat wxyz; opacities (N,1) in [0,1]; + sh (N,K,3) SH coefficients. Activated values are inverted to the standard 3D gaussian splat storage convention + (log scale, logit opacity). + """ + xyz = positions.cpu().numpy().astype(np.float32) + n = xyz.shape[0] + if n == 0: + raise ValueError("SplatToFile3D: gaussian is empty") + normals = np.zeros_like(xyz) + f = sh.cpu().numpy().astype(np.float32) # (N, K, 3) + f_dc = f[:, 0, :] # (N, 3) + f_rest = f[:, 1:, :].transpose(0, 2, 1).reshape(n, -1) # (N, 3*(K-1)) channel-major + op = opacities.cpu().numpy().astype(np.float32).reshape(n, 1).clip(1e-6, 1 - 1e-6) + op = np.log(op / (1.0 - op)) # inverse sigmoid (logit) + scale = np.log(scales.cpu().numpy().astype(np.float32).clip(min=1e-8)) + rot = rotations.cpu().numpy().astype(np.float32) # (N, 4) + + attrs = (['x', 'y', 'z', 'nx', 'ny', 'nz'] + + [f'f_dc_{i}' for i in range(3)] + + [f'f_rest_{i}' for i in range(f_rest.shape[1])] + + ['opacity'] + [f'scale_{i}' for i in range(3)] + [f'rot_{i}' for i in range(4)]) + elements = np.empty(n, dtype=[(a, 'f4') for a in attrs]) + elements[:] = list(map(tuple, np.concatenate([xyz, normals, f_dc, f_rest, op, scale, rot], axis=1))) + + header = "ply\nformat binary_little_endian 1.0\n" + f"element vertex {n}\n" + header += "".join(f"property float {a}\n" for a in attrs) + "end_header\n" + return header.encode('ascii') + elements.tobytes() + + +# .ksplat (mkkellogg SplatBuffer) level 0, SH degree 0: 4096-byte header, one 1024-byte section header, +# then N 44-byte records. Bucketing/quantization only exist at levels >= 1. See SplatBuffer.js. +_KSPLAT_HEADER_BYTES = 4096 +_KSPLAT_SECTION_HEADER_BYTES = 1024 +_KSPLAT_BYTES_PER_SPLAT = 44 # center 12 + scale 12 + rotation 16 + color(RGBA u8) 4 +_KSPLAT_VERSION = (0, 1) # SplatBuffer CurrentMajor/MinorVersion + + +def _gaussian_ksplat_bytes(positions, scales, rotations, opacities, sh) -> bytes: + """Serialize gaussian tensors as a level-0, SH degree-0 .ksplat (linear scale, opacity in color alpha). + + positions (N,3) world; scales (N,3) linear; rotations (N,4) wxyz; opacities (N,1) in [0,1]; sh (N,K,3). + """ + xyz = positions.cpu().numpy().astype(np.float32) + n = xyz.shape[0] + if n == 0: + raise ValueError("SplatToFile3D: gaussian is empty") + scale = scales.cpu().numpy().astype(np.float32) + rot = rotations.cpu().numpy().astype(np.float32) # wxyz, mirrors the .ply rot order + rot = rot / np.linalg.norm(rot, axis=1, keepdims=True).clip(1e-12) + rgb = np.clip(sh[:, 0, :].cpu().numpy().astype(np.float32) * _C0 + 0.5, 0, 1) + op = opacities.cpu().numpy().astype(np.float32).reshape(n, 1).clip(0, 1) + rgba = np.round(np.concatenate([rgb, op], axis=1) * 255.0).astype(np.uint8) # (N, 4) RGBA + + # 44-byte record: float center(3) + scale(3) + rot(4), then uint8 rgba(4). + floats = np.concatenate([xyz, scale, rot], axis=1).astype(' bytes: + """Serialize gaussian tensors as a gzip-compressed .spz (Niantic v2, SH degree 0, base color only). + + positions (N,3) world; scales (N,3) linear; rotations (N,4) wxyz; opacities (N,1) in [0,1]; sh (N,K,3). + """ + xyz = positions.cpu().numpy().astype(np.float32) + n = xyz.shape[0] + if n == 0: + raise ValueError("SplatToFile3D: gaussian is empty") + + # Positions: fixed point, masked to 24 bits, little-endian 3-byte words. + fixed = 1 << _SPZ_FRACTIONAL_BITS + qi = np.clip(np.round(xyz * fixed), -(1 << 23), (1 << 23) - 1).astype(np.int32) + qu = (qi & 0xFFFFFF).astype(np.uint32) + pos = np.stack([qu & 0xFF, (qu >> 8) & 0xFF, (qu >> 16) & 0xFF], axis=-1).reshape(n, 9).astype(np.uint8) + + alpha = np.round(opacities.cpu().numpy().astype(np.float32).reshape(n) * 255.0).clip(0, 255).astype(np.uint8) + + rgb = sh[:, 0, :].cpu().numpy().astype(np.float32) * _C0 + 0.5 + col = np.round(((rgb - 0.5) / _SPZ_COLOR_SCALE + 0.5) * 255.0).clip(0, 255).astype(np.uint8) # (N,3) + + sln = np.log(scales.cpu().numpy().astype(np.float32).clip(min=1e-9)) + scb = np.round((sln + 10.0) * 16.0).clip(0, 255).astype(np.uint8) # (N,3) inverts exp(b/16-10) + + rot = rotations.cpu().numpy().astype(np.float32) # wxyz + rot = rot / np.linalg.norm(rot, axis=1, keepdims=True).clip(1e-12) + rot[rot[:, 0] < 0] *= -1.0 # canonical w >= 0 (w dropped on decode) + rotb = np.round((rot[:, 1:4] + 1.0) * 127.5).clip(0, 255).astype(np.uint8) # (N,3) x,y,z + + header = bytearray(16) + struct.pack_into(' (positions, scales linear, rotations wxyz, opacities [0,1], sh (N,K,3)) ---- +# Inverse of the writers above and of spark's loaders. ksplat/splat/spz carry base color only (SH degree 0 +# -> K=1); .ply round-trips full SH. None of the formats flip axes, so import is the identity of export. +_PLY_DTYPES = {'char': 'i1', 'uchar': 'u1', 'short': 'i2', 'ushort': 'u2', 'int': 'i4', 'uint': 'u4', + 'float': 'f4', 'double': 'f8', 'int8': 'i1', 'uint8': 'u1', 'int16': 'i2', 'uint16': 'u2', + 'int32': 'i4', 'uint32': 'u4', 'float32': 'f4', 'float64': 'f8'} +_KSPLAT_COMPRESSION = { # level -> (bytesPerCenter, scale, rotation, color, shComponent, defaultScaleRange) + 0: (12, 12, 16, 4, 4, 1), 1: (6, 6, 8, 4, 2, 32767), 2: (6, 6, 8, 4, 1, 32767)} +_KSPLAT_SH_COMPONENTS = {0: 0, 1: 9, 2: 24, 3: 45} + + +def _rgb_to_sh_dc(rgb): + return ((np.asarray(rgb, np.float32) - 0.5) / _C0)[:, None, :] # (N,3) base color -> (N,1,3) SH DC + + +def _norm_quat(q): + return q / np.linalg.norm(q, axis=1, keepdims=True).clip(1e-12) + + +def _parse_ply_gaussian(data: bytes): + end = data.find(b'end_header') + if end < 0: + raise ValueError("File3DToSplat: not a PLY (missing end_header)") + header = data[:end].decode('ascii', 'replace') + body = end + len(b'end_header') + body += 2 if data[body:body + 2] == b'\r\n' else 1 + count, props, in_vertex = 0, [], False + for line in header.splitlines(): + p = line.split() + if not p: + continue + if p[0] == 'format' and p[1] != 'binary_little_endian': + raise ValueError(f"File3DToSplat: unsupported PLY format '{p[1]}' (need binary_little_endian)") + if p[0] == 'element': + in_vertex = p[1] == 'vertex' + if in_vertex: + count = int(p[2]) + elif p[0] == 'property' and in_vertex: + if p[1] == 'list': + raise ValueError("File3DToSplat: PLY vertex has list properties (unsupported)") + props.append((p[2], '<' + _PLY_DTYPES[p[1]])) + arr = np.frombuffer(data, np.dtype(props), count=count, offset=body) + names = arr.dtype.names + c = lambda k: arr[k].astype(np.float32) + n = count + + xyz = np.stack([c('x'), c('y'), c('z')], 1) + if 'scale_0' in names: + scale = np.exp(np.stack([c('scale_0'), c('scale_1'), c('scale_2')], 1)) # 3DGS stores log scale + else: + scale = np.full((n, 3), 0.01, np.float32) + if 'rot_0' in names: + rot = _norm_quat(np.stack([c('rot_0'), c('rot_1'), c('rot_2'), c('rot_3')], 1)) # wxyz + else: + rot = np.tile(np.array([1, 0, 0, 0], np.float32), (n, 1)) + opacity = 1.0 / (1.0 + np.exp(-c('opacity'))) if 'opacity' in names else np.ones(n, np.float32) + + if 'f_dc_0' in names: + dc = np.stack([c('f_dc_0'), c('f_dc_1'), c('f_dc_2')], 1) # (N,3) + rest = sorted((k for k in names if k.startswith('f_rest_')), key=lambda s: int(s.split('_')[-1])) + if rest: + r = np.stack([c(k) for k in rest], 1) # (N, 3*(K-1)) channel-major + kk = r.shape[1] // 3 + 1 + r = r.reshape(n, 3, kk - 1).transpose(0, 2, 1) # -> (N, K-1, 3) + sh = np.concatenate([dc[:, None, :], r], 1) + else: + sh = dc[:, None, :] + elif 'red' in names: + sh = _rgb_to_sh_dc(np.stack([c('red'), c('green'), c('blue')], 1) / 255.0) + else: + sh = np.zeros((n, 1, 3), np.float32) + return xyz, scale, rot, opacity, sh + + +def _parse_splat_gaussian(data: bytes): + # antimatter15 .splat: 32-byte records (f32 xyz, f32 scale, u8 rgba, u8 quat as (b-128)/128 wxyz). + if len(data) % 32 != 0: + raise ValueError("File3DToSplat: .splat size is not a multiple of 32 bytes") + rec = np.frombuffer(data, np.dtype([('xyz', ' 0: + ct, ft = (' full_splats: + lengths = np.frombuffer(data, '> 30) & 3 + q = np.zeros((n, 4), np.float32) # x,y,z,w + remaining, sumsq = combined.copy(), np.zeros(n, np.float64) + for comp in (3, 2, 1, 0): + active = comp != largest + value = (remaining & 0x1FF).astype(np.float64) + sign = (remaining >> 9) & 1 + remaining = np.where(active, remaining >> 10, remaining) + val = (1.0 / math.sqrt(2)) * (value / 0x1FF) + val = np.where(sign == 1, -val, val) + q[active, comp] = val[active] + sumsq += np.where(active, val * val, 0.0) + q[np.arange(n), largest] = np.sqrt(np.clip(1.0 - sumsq, 0, None)) + rot = _norm_quat(np.stack([q[:, 3], q[:, 0], q[:, 1], q[:, 2]], 1)) # xyzw -> wxyz + else: + qb = np.frombuffer(raw, np.uint8, count=n * 3, offset=off).reshape(n, 3).astype(np.float32) + xq = qb / 127.5 - 1.0 + w = np.sqrt(np.clip(1.0 - (xq ** 2).sum(1), 0, None)) + rot = _norm_quat(np.concatenate([w[:, None], xq], 1)) # wxyz + return xyz, scale, rot, alpha, _rgb_to_sh_dc(rgb) + + +_GAUSSIAN_PARSERS = {"ply": _parse_ply_gaussian, "splat": _parse_splat_gaussian, + "ksplat": _parse_ksplat_gaussian, "spz": _parse_spz_gaussian} + + +def _detect_splat_format(data: bytes) -> str: + if data[:3] == b'ply': + return "ply" + if data[:2] == b'\x1f\x8b': # gzip -> spz + return "spz" + if len(data) >= 2 and data[0] == 0 and data[1] >= 1: # ksplat version 0.x header + return "ksplat" + if len(data) % 32 == 0: + return "splat" + raise ValueError("File3DToSplat: could not determine splat format from contents") + + +def _gaussian_item(g: Types.SPLAT, i: int, device): + # Slice batch item i to its real length, as float32 torch tensors on `device` (SH DC -> base RGB). + end = _real_len(g, i) + to = lambda a: a.to(device=device, dtype=torch.float32) + xyz = to(g.positions[i, :end]) + rgb = (to(g.sh[i, :end, 0, :]) * _C0 + 0.5).clamp(0, 1) + opacity = to(g.opacities[i, :end]).reshape(-1) + scale = to(g.scales[i, :end]) + rot = to(g.rotations[i, :end]) + return xyz, rgb, opacity, scale, rot + + +def _quat_to_mat(q): + # q: (N, 4) wxyz, normalized -> (N, 3, 3) + q = q / q.norm(dim=-1, keepdim=True).clamp_min(1e-12) + w, x, y, z = q.unbind(-1) + return torch.stack([ + 1 - 2 * (y * y + z * z), 2 * (x * y - w * z), 2 * (x * z + w * y), + 2 * (x * y + w * z), 1 - 2 * (x * x + z * z), 2 * (y * z - w * x), + 2 * (x * z - w * y), 2 * (y * z + w * x), 1 - 2 * (x * x + y * y), + ], dim=-1).reshape(-1, 3, 3) + + +def _quat_mul(a, b): + # Hamilton product a (x) b, wxyz. + aw, ax, ay, az = a.unbind(-1) + bw, bx, by, bz = b.unbind(-1) + return torch.stack([ + aw * bw - ax * bx - ay * by - az * bz, + aw * bx + ax * bw + ay * bz - az * by, + aw * by - ax * bz + ay * bw + az * bx, + aw * bz + ax * by - ay * bx + az * bw, + ], dim=-1) + + +def _euler_to_quat(rx, ry, rz): + # Degrees, applied as Rz @ Ry @ Rx (rotate about X, then Y, then Z in world). Returns wxyz. + c, s = np.cos(np.radians([rx, ry, rz]) / 2.0), np.sin(np.radians([rx, ry, rz]) / 2.0) + qx = torch.tensor([c[0], s[0], 0.0, 0.0], dtype=torch.float32) + qy = torch.tensor([c[1], 0.0, s[1], 0.0], dtype=torch.float32) + qz = torch.tensor([c[2], 0.0, 0.0, s[2]], dtype=torch.float32) + return _quat_mul(_quat_mul(qz, qy), qx) + + +def _mat_to_quat(m): + # Rotation matrix (..., 3, 3) -> quaternion (..., 4) wxyz. Batched; builds the four candidate quaternions + # and keeps the one with the largest component (numerically stable across all rotations). + m00, m11, m22 = m[..., 0, 0], m[..., 1, 1], m[..., 2, 2] + m21, m12 = m[..., 2, 1], m[..., 1, 2] + m02, m20 = m[..., 0, 2], m[..., 2, 0] + m10, m01 = m[..., 1, 0], m[..., 0, 1] + q2 = torch.stack([1 + m00 + m11 + m22, 1 + m00 - m11 - m22, + 1 - m00 + m11 - m22, 1 - m00 - m11 + m22], -1) # 4 * (w^2, x^2, y^2, z^2) + cand = torch.stack([ + torch.stack([q2[..., 0], m21 - m12, m02 - m20, m10 - m01], -1), + torch.stack([m21 - m12, q2[..., 1], m10 + m01, m02 + m20], -1), + torch.stack([m02 - m20, m10 + m01, q2[..., 2], m12 + m21], -1), + torch.stack([m10 - m01, m02 + m20, m12 + m21, q2[..., 3]], -1), + ], -2) # (...,4,4) candidates, rows = wxyz + sel = q2.argmax(-1) + q = torch.gather(cand, -2, sel[..., None, None].expand(sel.shape + (1, 4)))[..., 0, :] + return q / q.norm(dim=-1, keepdim=True).clamp_min(1e-12) + + +class SplatToFile3D(IO.ComfyNode): + FORMAT_WRITERS = { + "ply": _gaussian_ply_bytes, + "ksplat": _gaussian_ksplat_bytes, + "spz": _gaussian_spz_bytes, + } + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="SplatToFile3D", + display_name="Create 3D File (from Splat)", + search_aliases=["gaussian to ply", "splat to file", "export gaussian"], + category="3d/splat", + description="Serialize a gaussian splat to a File3D object for Save / Preview 3D nodes. " + "Supports one item per batch only.", + inputs=[ + IO.Splat.Input("splat"), + IO.Combo.Input("format", options=list(cls.FORMAT_WRITERS), # TODO: add "splat" when we have a writer for it + tooltip="ply: standard 3D Gaussian Splat with full spherical harmonics. " + "ksplat: mkkellogg SplatBuffer (level 0, uncompressed), base color only " + "spz: Niantic gzip-compressed (~10x smaller), base color only " + ), + ], + outputs=[IO.File3DSplatAny.Output(display_name="model_3d")], + ) + + @classmethod + def execute(cls, splat, format="ply") -> IO.NodeOutput: + writer = cls.FORMAT_WRITERS.get(format) + if writer is None: + raise ValueError(f"Unsupported splat format: {format!r}") + + if splat.positions.shape[0] > 1: + logging.warning("SplatToFile3D supports one item per batch only. Got %d; using first.", splat.positions.shape[0]) + end = _real_len(splat, 0) + data = writer(splat.positions[0, :end], splat.scales[0, :end], + splat.rotations[0, :end], splat.opacities[0, :end], splat.sh[0, :end]) + return IO.NodeOutput(Types.File3D(BytesIO(data), file_format=format)) + + +class File3DToSplat(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="File3DToSplat", + display_name="Get Splat", + search_aliases=["load splat", "ply to splat", "import splat", "file to splat"], + category="3d/splat", + description="Parse a splat File3D into a gaussian splat. Inverse of Create 3D File (from Splat). " + "Supported format: PLY, SPLAT, KSPLAT, SPZ. PLY carries full spherical harmonics, " + "the other formats are base color only. Format is auto-detected from the file contents.", + inputs=[ + IO.MultiType.Input( + IO.File3DAny.Input("model_3d"), + types=[IO.File3DSplatAny, IO.File3DPLY, IO.File3DSPLAT, IO.File3DKSPLAT, IO.File3DSPZ], + tooltip="A gaussian splat 3D file", + ), + ], + outputs=[IO.Splat.Output(display_name="splat")], + ) + + @classmethod + def execute(cls, model_3d: Types.File3D) -> IO.NodeOutput: + data = model_3d.get_bytes() + fmt = (model_3d.format or "").lower() + parser = _GAUSSIAN_PARSERS.get(fmt) or _GAUSSIAN_PARSERS[_detect_splat_format(data)] + xyz, scale, rot, opacity, sh = parser(data) + + t = lambda a: torch.from_numpy(np.ascontiguousarray(a)).float() + splat = Types.SPLAT( + t(xyz)[None], # (1, N, 3) + t(scale)[None], # (1, N, 3) linear + t(rot)[None], # (1, N, 4) wxyz + t(opacity).reshape(1, -1, 1), # (1, N, 1) + t(sh)[None], # (1, N, K, 3) + ) + return IO.NodeOutput(splat) + + +def _view_matrix_t(yaw_deg, pitch_deg, device): + y, p = math.radians(yaw_deg), math.radians(pitch_deg) + cy, sy, cp, sp = math.cos(y), math.sin(y), math.cos(p), math.sin(p) + Ry = torch.tensor([[cy, 0, sy], [0, 1, 0], [-sy, 0, cy]], device=device) + Rx = torch.tensor([[1, 0, 0], [0, cp, -sp], [0, sp, cp]], device=device) + return Rx @ Ry + + +def _camera_basis(camera_info, dev): + # Look-at basis in the splat frame, named by their projection rows: right = image +x, up = image +y + # (down, since yflip=1), fwd = view/depth axis (eye -> scene). Load3D is three.js (right-handed, Y-up, + # camera looks down -Z); the splat is 3DGS (Y-down, Z-forward). World -> splat is a 180 deg rotation + # about X: (x, y, z) -> (x, -y, -z) (det +1, no mirror, no axis swap). + pos, tgt = camera_info.get("position", {}), camera_info.get("target", {}) + m = lambda d: torch.tensor([float(d.get("x", 0.0)), -float(d.get("y", 0.0)), -float(d.get("z", 0.0))], device=dev) + eye, target = m(pos), m(tgt) + mv = lambda v: torch.stack([v[0], -v[1], -v[2]]) # same world->splat map, for direction vectors + n = lambda v: v / v.norm().clamp_min(1e-8) + q = camera_info.get("quaternion") + if q: # exact camera world rotation (incl. roll) + qwxyz = torch.tensor([float(q.get("w", 1.0)), float(q.get("x", 0.0)), + float(q.get("y", 0.0)), float(q.get("z", 0.0))], device=dev) + R = _quat_to_mat(qwxyz[None])[0] # columns = camera world axes; looks down local -Z + right = n(mv(R[:, 0])) # camera +X -> image right + up = n(mv(-R[:, 1])) # camera +Y is image up; image-down row is its negative + fwd = n(mv(-R[:, 2])) # camera looks down local -Z -> view direction + return eye, target, right, up, fwd + fwd = n(target - eye) # no quaternion: orbit-consistent, roll-free + yaw = math.degrees(math.atan2(-float(fwd[0]), float(fwd[2]))) + pitch = math.degrees(math.asin(max(-1.0, min(1.0, float(fwd[1]))))) + W = _view_matrix_t(yaw, pitch, dev) + return eye, target, W[0], W[1], W[2] + + +def _lookat_quat_wxyz(position, target, dev): + # three.js lookAt in world frame: camera local +Z = (eye - target), up = world +Y. Returns wxyz. + z = position - target + z = z / z.norm().clamp_min(1e-8) + up0 = torch.tensor([0.0, 1.0, 0.0], device=dev) + if z.dot(up0).abs() > 0.999: # looking straight up/down + up0 = torch.tensor([0.0, 0.0, 1.0], device=dev) + x = torch.linalg.cross(up0, z) + x = x / x.norm().clamp_min(1e-8) + y = torch.linalg.cross(z, x) + R = torch.stack([x, y, z], dim=1) # columns = camera world axes + return _mat_to_quat(R[None])[0] + + +def _lookat_camera_info(position, target, fov, dev, zoom=1.0, camera_type="perspective", roll=0.0): + # Build a camera_info from a world-space (right-handed, Y-up) eye + look-at target; up = world +Y. + pos = torch.as_tensor(position, dtype=torch.float32, device=dev) + tgt = torch.as_tensor(target, dtype=torch.float32, device=dev) + q = _lookat_quat_wxyz(pos, tgt, dev) + if roll: # roll about the view axis (camera local Z) + a = math.radians(roll) + qz = torch.tensor([math.cos(a / 2), 0.0, 0.0, math.sin(a / 2)], device=dev) + q = _quat_mul(q[None], qz[None])[0] + xyz = lambda v: {"x": float(v[0]), "y": float(v[1]), "z": float(v[2])} + return {"position": xyz(pos), "target": xyz(tgt), + "quaternion": {"x": float(q[1]), "y": float(q[2]), "z": float(q[3]), "w": float(q[0])}, + "fov": float(fov), "cameraType": str(camera_type), "zoom": float(zoom)} + + +def _quat_camera_info(position, quat_xyzw, fov, dev, zoom=1.0, camera_type="perspective"): + # camera_info from an explicit world position + camera-rotation quaternion (three.js: looks down local -Z). + pos = torch.as_tensor(position, dtype=torch.float32, device=dev) + qx, qy, qz, qw = (float(c) for c in quat_xyzw) + qwxyz = torch.tensor([qw, qx, qy, qz], dtype=torch.float32, device=dev) + qwxyz = qwxyz / qwxyz.norm().clamp_min(1e-8) + R = _quat_to_mat(qwxyz[None])[0] + tgt = pos - R[:, 2] # look one unit down local -Z + xyz = lambda v: {"x": float(v[0]), "y": float(v[1]), "z": float(v[2])} + return {"position": xyz(pos), "target": xyz(tgt), + "quaternion": {"x": float(qwxyz[1]), "y": float(qwxyz[2]), "z": float(qwxyz[3]), "w": float(qwxyz[0])}, + "fov": float(fov), "cameraType": str(camera_type), "zoom": float(zoom)} + + +def _orbit_camera_info(yaw, pitch, distance, fov, pivot_splat, dev): + # Orbit helper for RenderSplat's default camera: yaw/pitch about `pivot_splat` (splat frame) at `distance`. + # World<->splat is the (x,-y,-z) map, so _camera_basis recovers exactly _view_matrix_t(yaw, pitch). + y, p = math.radians(yaw), math.radians(pitch) + cy, sy, cp, sp = math.cos(y), math.sin(y), math.cos(p), math.sin(p) + fwd_splat = torch.tensor([-cp * sy, sp, cp * cy], device=dev) # == _view_matrix_t(yaw, pitch)[2] + m = lambda v: torch.stack([v[0], -v[1], -v[2]]) # splat<->world (its own inverse) + return _lookat_camera_info(m(pivot_splat - distance * fwd_splat), m(pivot_splat), fov, dev) + + +def _orbit_camera_info_yaw(camera_info, angle_deg, dev): + # Turntable: rigidly rotate a camera_info about world +Y around its target by angle_deg. Returns a new dict. + a = math.radians(angle_deg) + ca, sa = math.cos(a), math.sin(a) + v = lambda d: torch.tensor([float(d.get("x", 0.0)), float(d.get("y", 0.0)), float(d.get("z", 0.0))], device=dev) + pos, tgt = v(camera_info.get("position", {})), v(camera_info.get("target", {})) + Ry = torch.tensor([[ca, 0.0, sa], [0.0, 1.0, 0.0], [-sa, 0.0, ca]], device=dev) + new_pos = tgt + Ry @ (pos - tgt) + q = camera_info.get("quaternion") or {} + qcur = torch.tensor([float(q.get("w", 1.0)), float(q.get("x", 0.0)), + float(q.get("y", 0.0)), float(q.get("z", 0.0))], device=dev) + qy = torch.tensor([math.cos(a / 2), 0.0, math.sin(a / 2), 0.0], device=dev) # world +Y rotation + qn = _quat_mul(qy[None], qcur[None])[0] + xyz = lambda t: {"x": float(t[0]), "y": float(t[1]), "z": float(t[2])} + return {**camera_info, "position": xyz(new_pos), + "quaternion": {"x": float(qn[1]), "y": float(qn[2]), "z": float(qn[3]), "w": float(qn[0])}} + + +def _gauss_blur(x, sigma, dev): + # Separable Gaussian blur of (1, C, H, W). Used to denoise the screen-space normal map. + r = max(1, int(round(3 * sigma))) + k = torch.exp(-0.5 * (torch.arange(-r, r + 1, device=dev, dtype=torch.float32) / sigma) ** 2) + k = k / k.sum() + c = x.shape[1] + x = torch.nn.functional.conv2d(x, k.view(1, 1, 1, -1).expand(c, 1, 1, -1), padding=(0, r), groups=c) + x = torch.nn.functional.conv2d(x, k.view(1, 1, -1, 1).expand(c, 1, -1, 1), padding=(r, 0), groups=c) + return x + + +def _render_gaussian(xyz, rgb, opacity, scale, rot, width, height, splat_scale, bg, camera_info, + sharpen=1.0, headlight_shading=0.0, render_style="color"): + # Perspective-correct anisotropic gaussian splat rasterizer. Each splat is weighted by its 3D Gaussian's + # peak along each pixel's ray (AAA / Hahlbohm), composited front-to-back across depth slabs. `render_style` + # selects the image: color / clay / depth / normal. Returns (image HxWx3, coverage mask HxW) on CPU. + dev = comfy.model_management.get_torch_device() + t = lambda a: torch.as_tensor(a, dtype=torch.float32, device=dev) + idev, idtype = comfy.model_management.intermediate_device(), comfy.model_management.intermediate_dtype() + xyz, rgb, opacity = t(xyz), t(rgb).clamp(0, 1), t(opacity).reshape(-1) + scale, rot = t(scale) * float(splat_scale), t(rot) + do_linear = render_style == "color" # colour blends in linear light, re-encoded at the end + if do_linear: + rgb = _srgb_to_linear(rgb) + flat = width * height + bg_t = t(bg) + bg_comp = _srgb_to_linear(bg_t) if do_linear else bg_t # background blended in the same space as the splats + need_depth = render_style == "depth" + need_normal = render_style in ("normal", "clay") or headlight_shading > 0 + + def background_only(): # no splats to rasterize -> just the background + empty mask + img = bg_t.expand(height, width, 3) if render_style == "color" else torch.zeros(height, width, 3, device=dev) + return img.to(idev, idtype), torch.zeros(height, width, device=idev, dtype=idtype) + + if xyz.shape[0] == 0: # empty input (e.g. all culled by opacity_threshold) + return background_only() + + eye, target, right, up, fwd = _camera_basis(camera_info, dev) # all camera state comes from camera_info + W = torch.stack([right, up, fwd], 0) # rows = camera axes (world -> camera) + cam = (xyz - eye) @ W.T + fov = float(camera_info.get("fov", 0) or 0) or 35.0 + zoom = float(camera_info.get("zoom", 1.0) or 1.0) # three.js digital zoom: scales the focal length + is_ortho = str(camera_info.get("cameraType", "")).lower().startswith("ortho") + xc, yc, zc = cam.unbind(-1) + + keep = zc > 1e-2 + xc, yc, zc, rgb, opacity, scale, rot = (a[keep] for a in (xc, yc, zc, rgb, opacity, scale, rot)) + if xc.shape[0] == 0: # nothing in front of the camera -> background only + return background_only() + if render_style == "clay": + rgb = torch.full_like(rgb, 0.75) # neutral albedo -> shading shows pure geometry + + f = (min(width, height) / 2) / math.tan(math.radians(fov) / 2) * zoom # fov over the smaller axis, x camera zoom + cx0, cy0 = width / 2, height / 2 + + # Camera-space 3D covariance per splat: Sigma = (W Rq) diag(scale^2) (W Rq)^T, plus a tiny relative + # regularizer for a stable inverse (a pixel-size Mip low-pass would over-thicken flat surfels and blur). + Mw = W[None] @ _quat_to_mat(rot) # (N,3,3) world -> camera + cam_cov = (Mw * scale.square()[:, None, :]) @ Mw.transpose(1, 2) + cam_cov = cam_cov + (cam_cov.diagonal(dim1=-2, dim2=-1).mean(-1) * 1e-3)[:, None, None] * torch.eye(3, device=dev) + + # Perspective-correct weighting: peak of the 3D Gaussian along each pixel ray. Precompute Si, Si@mu, mu^T Si mu. + mu = torch.stack([xc, yc, zc], -1) + si = torch.linalg.inv(cam_cov) + simu = (si @ mu[:, :, None])[:, :, 0] # (N,3) + musimu = (mu * simu).sum(-1) # (N,) + s00, s01, s02 = si[:, 0, 0], si[:, 0, 1], si[:, 0, 2] + s11, s12, s22 = si[:, 1, 1], si[:, 1, 2], si[:, 2, 2] + simu0, simu1, simu2 = simu.unbind(-1) + if need_normal: # surfel normal = thinnest axis, oriented toward camera + nrm = Mw[torch.arange(Mw.shape[0], device=dev), :, scale.argmin(-1)] # (N,3) camera-space normal + nrm = nrm * torch.where(nrm[:, 2:3] > 0, -1.0, 1.0) # flip so nz <= 0 (faces camera) + + # Screen centre (exact) + footprint radius from the affine 2D projection (used only to size the kernel). + # The image is +y-down, so the projection's y row is unflipped - it matches the splat frame's +Y. + jm = torch.zeros(xc.shape[0], 2, 3, device=dev) + if is_ortho: # parallel projection: screen = s * (xc, yc) + s = f / float((target - eye).norm().clamp_min(1e-6)) # pixels per world unit at the target plane + cx, cy = cx0 + s * xc, cy0 + s * yc + jm[:, 0, 0] = s + jm[:, 1, 1] = s + else: # perspective: screen = f * (xc, yc) / zc + invz = 1.0 / zc + cx, cy = cx0 + f * xc * invz, cy0 + f * yc * invz + jm[:, 0, 0], jm[:, 0, 2] = f * invz, -f * xc * invz.square() + jm[:, 1, 1], jm[:, 1, 2] = f * invz, -f * yc * invz.square() + cov2 = jm @ cam_cov @ jm.transpose(1, 2) + a, b, c = cov2[:, 0, 0], cov2[:, 0, 1], cov2[:, 1, 1] + max_eig = (a + c) * 0.5 + (((a - c) * 0.5).square() + b * b).clamp_min(0).sqrt() + radius = 3.0 * max_eig.clamp_min(1e-8).sqrt() + K = int(min(max(24, min(width, height) // 16), max(2, math.ceil(_quantile(radius, 0.995).item())))) + + # Per-splat kernel size: bucket splats by radius into a coarse ladder of window sizes (global K stays the cap) so + # small splats (the bulk of it) use a small window. + levels = [L for L in (16, 64, 256) if L < K] + [K] + levels_t = torch.tensor(levels, device=dev, dtype=torch.float32) + grids = [] + for L in levels: + rng = torch.arange(-L, L + 1, device=dev, dtype=torch.float32) + gy, gx = torch.meshgrid(rng, rng, indexing="ij") + grids.append((gx.reshape(-1), gy.reshape(-1))) + blevel = torch.bucketize(radius * (4.0 / 3.0), levels_t).clamp_(max=len(levels) - 1) # window >= ~4 sigma + + n = zc.shape[0] + ns = int(min(256, max(1, n // 1000))) # depth slabs: 1 per ~1000 splats, capped + nl = len(levels) + order = torch.argsort(zc) # front (small zc) -> back -> defines the slabs + bounds = torch.linspace(0, n, ns + 1, device=dev).round().long() + rank = torch.empty(n, dtype=torch.long, device=dev) + rank[order] = torch.arange(n, device=dev) # depth rank of each splat + slab_id = (torch.searchsorted(bounds, rank, right=True) - 1).clamp_(0, ns - 1) + key = slab_id * nl + blevel # group by slab, then kernel level (order-free within) + order = torch.argsort(key) + key = key[order] + + cxr, cyr = cx[order].round(), cy[order].round() + s00, s01, s02 = s00[order], s01[order], s02[order] + s11, s12, s22 = s11[order], s12[order], s22[order] + s01b, s02b, s12b = s01 * 2, s02 * 2, s12 * 2 # doubled cross terms for the fused quadratic forms + simu0, simu1, simu2, musimu = simu0[order], simu1[order], simu2[order], musimu[order] + opacity, rgb = opacity[order], rgb[order] + zc_o = zc[order] if need_depth else None + nrm_o = nrm[order] if need_normal else None + mux_o, muy_o, muz_o = (xc[order], yc[order], zc[order]) if is_ortho else (None, None, None) + + # Pack the per-splat scalars into one tensor so each chunk slices once + common = [cxr, cyr, s00, s11, s22, s01b, s02b, s12b, opacity] + pstack = torch.stack(common + ([s02, s12, mux_o, muy_o, muz_o] if is_ortho else [simu0, simu1, simu2, musimu])) + + # Precompute the (slab, level) run table on-GPU and pull it to the CPU once + starts = torch.cat([torch.zeros(1, dtype=torch.long, device=dev), (key[1:] != key[:-1]).nonzero().flatten() + 1]) + ks = key[starts] + run_lo = starts.tolist() + [n] + run_lev = (ks % nl).tolist() + run_slab = torch.div(ks, nl, rounding_mode="floor").tolist() + slab_runs = [[] for _ in range(ns)] + for r in range(len(run_lev)): + slab_runs[run_slab[r]].append((run_lo[r], run_lo[r + 1], run_lev[r])) + + def splat(lo, hi, ox, oy): # -> pixel idx (m,M), alpha (m,M); weight = 3D Gaussian peak along each pixel's ray + cols = pstack[:, lo:hi, None].unbind(0) + cxr_, cyr_, a00, a11, a22, b01, b02, b12, opa = cols[:9] # a* = Si components; b* = 2 * cross terms + px = cxr_ + ox[None, :] + py = cyr_ + oy[None, :] + valid = (px >= 0) & (px < width) & (py >= 0) & (py < height) + if is_ortho: # parallel ray (0,0,1) from screen point (X, Y, 0); rz constant per splat + c02, c12, mx, my, mz = cols[9:] + rx = (px - cx0) / s - mx + ry = (py - cy0) / s - my + rz = -mz + a22rz = a22 * rz + inx = torch.addcmul(b02 * rz, a00, rx).addcmul_(b01, ry) # a00 rx + b01 ry + b02 rz + rSr = torch.addcmul(a22rz * rz, rx, inx).addcmul_(ry, torch.addcmul(b12 * rz, a11, ry)) + dsr = torch.addcmul(a22rz, c02, rx).addcmul_(c12, ry) + q = torch.addcdiv(rSr, dsr * dsr, a22.clamp_min(1e-12), value=-1).clamp_min_(0) + else: # perspective ray (dx,dy,1) through the camera origin + su0, su1, su2, mus = cols[9:] + dx, dy = (px - cx0) / f, (py - cy0) / f + dsid = torch.addcmul(a22, dx, torch.addcmul(b02, a00, dx)) # a22 + dx*(a00 dx + b02) + dsid = dsid.addcmul_(dy, torch.addcmul(b12, a11, dy)) # + dy*(a11 dy + b12) + dsid = dsid.addcmul_(b01 * dx, dy) # + (2 s01) dx dy + dsimu = torch.addcmul(su2, dx, su0).addcmul_(dy, su1) + q = torch.addcdiv(mus, dsimu * dsimu, dsid.clamp_min(1e-12), value=-1).clamp_min_(0) + alpha = (opa * torch.exp(-0.5 * q) * valid).clamp_(0, 0.999) + idx = py.long().clamp(0, height - 1) * width + px.long().clamp(0, width - 1) + return idx, alpha + + # Front-to-back compositing over the depth slabs set up above. Within a slab the accumulation is a pure + # sum (order-independent), so splats are grouped by kernel level and each level uses its own tight window. + sharp = sharpen != 1.0 # winner-take-more colour blend: dominant splat shows more + cacc = torch.zeros((flat, 3), device=dev) + trans = torch.ones((flat,), device=dev) + a_buf = torch.zeros((flat,), device=dev) # sum alpha -> colour/depth/normal weight (alpha-weighted mean) + tau_buf = torch.zeros((flat,), device=dev) # sum -ln(1-alpha) -> slab opacity = 1-prod(1-alpha) + crgb = torch.zeros((flat, 3), device=dev) # sum alpha^p * rgb -> slab colour + wbuf = torch.zeros((flat,), device=dev) if sharp else None # sum alpha^p -> colour normalizer (sharp only) + dacc = torch.zeros((flat,), device=dev) if need_depth else None # front-weighted depth + nacc = torch.zeros((flat, 3), device=dev) if need_normal else None # front-weighted camera-space normal + zslab = torch.zeros((flat,), device=dev) if need_depth else None + nslab = torch.zeros((flat, 3), device=dev) if need_normal else None + stale = 0 # consecutive fully-occluded slabs -> early-out + for si in range(ns): + runs = slab_runs[si] + if not runs: + continue + a_buf.zero_() + tau_buf.zero_() + crgb.zero_() + if sharp: + wbuf.zero_() + if need_depth: + zslab.zero_() + if need_normal: + nslab.zero_() + for r_lo, r_hi, li in runs: # contiguous same-kernel-level runs in this slab + ox, oy = grids[li] + ch = max(2048, 10_000_000 // ox.shape[0]) # splats/chunk, bounded by this level's kernel size + for lo in range(r_lo, r_hi, ch): + hi = min(lo + ch, r_hi) + idx, alpha = splat(lo, hi, ox, oy) + idx, af = idx.reshape(-1), alpha.reshape(-1) + a_buf.index_add_(0, idx, af) + tau_buf.index_add_(0, idx, (-torch.log1p(-alpha)).reshape(-1)) # -ln(1-alpha), correct opacity merge + apw = alpha.pow(sharpen) if sharp else alpha # bias colour toward the highest-alpha splat + crgb.index_add_(0, idx, (apw[:, :, None] * rgb[lo:hi, None, :]).reshape(-1, 3)) + if sharp: + wbuf.index_add_(0, idx, apw.reshape(-1)) + if need_depth: + zslab.index_add_(0, idx, (alpha * zc_o[lo:hi, None]).reshape(-1)) + if need_normal: + nslab.index_add_(0, idx, (alpha[:, :, None] * nrm_o[lo:hi, None, :]).reshape(-1, 3)) + slab_a = 1 - torch.exp(-tau_buf) # 1 - prod(1-alpha): true opacity of the slab's splats + front = trans * slab_a + denom = wbuf if sharp else a_buf + cacc.addcmul_(front[:, None], crgb / denom.clamp_min(1e-8)[:, None]) # cacc += front * (crgb/denom) + if need_depth or need_normal: + ainv = a_buf.clamp_min(1e-8) # alpha-weighted-mean normalizer (depth/normal only) + if need_depth: + dacc.addcmul_(front, zslab / ainv) + if need_normal: + nacc.addcmul_(front[:, None], nslab / ainv[:, None]) + trans.mul_(1 - slab_a) + if si % 8 == 7: # checkpoint every 8 slabs (a per-slab GPU sync would cost more) + if float(front.max()) < 1e-3: # this checkpoint slab is fully occluded by what is in front + stale += 1 + if stale >= 2: # two occluded checkpoints running -> the rest are too -> stop + break + else: + stale = 0 + + cov = 1 - trans + covg = cov.reshape(height, width) + covm = covg > 0.5 if render_style in ("depth", "normal") else None # silhouette mask (depth/normal styles only) + depth_map = (dacc / cov.clamp_min(1e-6)).reshape(height, width) if need_depth else None + nrm_map = None + if need_normal: + # Per-splat surfel normals are jittery, so do a masked blur + nb = nacc.reshape(height, width, 3).permute(2, 0, 1)[None] + cb = cov.reshape(1, 1, height, width) + nb, cb = _gauss_blur(nb, 1.2, dev), _gauss_blur(cb, 1.2, dev) + normal = (nb / cb.clamp_min(1e-6))[0].permute(1, 2, 0) + nrm_map = normal / normal.norm(dim=-1, keepdim=True).clamp_min(1e-6) + + if render_style == "depth": # near = bright, far = dark, 0 off-object + d = torch.zeros(height, width, device=dev) + if bool(covm.any()): + lo, hi = depth_map[covm].min(), depth_map[covm].max() + d = torch.where(covm, ((hi - depth_map) / (hi - lo).clamp_min(1e-6)).clamp(0, 1), d) + img = d[:, :, None].expand(height, width, 3) + elif render_style == "normal": # OpenGL normal map: +X right, +Y up, +Z to viewer + enc = (nrm_map * t([1.0, -1.0, -1.0]) * 0.5 + 0.5).clamp(0, 1) + img = enc * covm[:, :, None] + else: # color / clay + img = cacc.reshape(height, width, 3) + if render_style == "clay": # studio key light + ambient -> sculpted matte look + kl = t([-0.4, -0.7, -0.6]) # key from screen upper-left, angled toward the viewer + kl = kl / kl.norm() + hl = (0.5 * (nrm_map * kl).sum(-1) + 0.5).clamp(0, 1) # half-Lambert: soft terminator, no harsh dark side + img = img * (0.35 + 0.65 * hl * hl)[:, :, None] # ambient floor + diffuse key + elif headlight_shading > 0: # camera headlight: darken faces turned from view + k = float(headlight_shading) + ndotl = (-nrm_map[:, :, 2]).clamp(0, 1) + img = img * (1 - 0.6 * k + 0.6 * k * ndotl)[:, :, None] + img = img.addcmul_(trans.reshape(height, width, 1), bg_comp) + if do_linear: # back to display space after linear compositing + img = _linear_to_srgb(img) + return img.clamp(0, 1).to(idev, idtype), covg.clamp(0, 1).to(idev, idtype) + + +class RenderSplat(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="RenderSplat", + display_name="Render Splat", + search_aliases=["splat to image", "render splat", "gaussian turntable"], + category="3d/splat", + description="Render a gaussian splat as an image with an anisotropic EWA rasterizer (oriented " + "elliptical splats, antialiased, depth-sorted front-to-back). The camera comes from a " + "camera_info input (Load / Preview 3D, or a Create Camera Info node); leave it empty to " + "auto-frame the splat. Set frames greater than 1 for a turntable batch of images to feed a Video node.", + inputs=[ + IO.Splat.Input("splat"), + IO.Int.Input("width", default=1024, min=64, max=2048, step=8), + IO.Int.Input("height", default=1024, min=64, max=2048, step=8), + IO.Int.Input("frames", default=1, min=-240, max=240, + tooltip="-1, 0, 1 = single still image; >1 = turntable, the camera orbits over a full " + "360 turn (works with any camera_info). Negative value orbits the other way."), + IO.Float.Input("splat_scale", default=1.0, min=0.1, max=5.0, step=0.05, advanced=True, + tooltip="Multiplier on each splat's projected footprint (lower = crisper points, " + "higher = softer/fuller surface)."), + IO.Float.Input("sharpen", default=2.0, min=1.0, max=8.0, step=0.5, + tooltip="Sharpen overlapping splats: 1.0 = physically-correct blend; higher biases " + "each pixel toward its dominant (nearest) splat for crisper texture, without " + "shrinking splats or opening gaps. Non-physical above 1."), + IO.Float.Input("headlight_shading", default=0.0, min=0.0, max=3.0, step=0.05, advanced=True, + tooltip="Diffuse shading from a light at the camera (headlight), using the splat surfel " + "normals: darkens surfaces that turn away from view to reveal form/curvature. " + "0 = flat albedo, 1 = strongest shading."), + IO.Float.Input("opacity_threshold", default=0.0, min=0.0, max=1.0, step=0.01, advanced=True, + tooltip="Cull gaussians with opacity below this (removes faint floaters)."), + IO.Combo.Input("render_style", options=["color", "clay", "depth", "normal"], + tooltip="What the image output shows: color, clay (neutral-albedo shaded), " + "depth (near=bright), normal (OpenGL normal map)."), + IO.Color.Input("background", default="#000000"), + IO.Image.Input("bg_image", optional=True, + tooltip="Optional background plate composited behind the splat (overrides the solid " + "background colour). Resized to the render size; a batch is used per frame, " + "a single image for all. color/clay only."), + IO.Load3DCamera.Input("camera_info", optional=True, + tooltip="Camera to render from - a Load3D / Preview3D camera or a Create Camera " + "Info node. If empty, the splat is auto-framed from a default 3/4 view."), + ], + outputs=[IO.Image.Output(display_name="image"), IO.Mask.Output(display_name="mask")], + ) + + @classmethod + def execute(cls, splat, width, height, frames, splat_scale, sharpen, headlight_shading, + opacity_threshold, background, render_style, camera_info=None, bg_image=None) -> IO.NodeOutput: + bg = _hex_to_rgb(background) + bg_imgs = None + if bg_image is not None: # resize the plate(s) to the render size: (B,H,W,3) + bi = bg_image[... , :3].movedim(-1, 1) # (B,3,H,W) + bi = comfy.utils.common_upscale(bi, width, height, "bicubic", "disabled") + bg_imgs = bi.movedim(1, -1).clamp(0, 1) + n_frames = abs(int(frames)) or 1 # magnitude = frame count (0 -> single still) + orbit_dir = -1.0 if frames < 0 else 1.0 # sign = orbit direction + imgs, masks = [], [] + device = comfy.model_management.get_torch_device() + total = splat.positions.shape[0] * n_frames + pbar = comfy.utils.ProgressBar(total) if total > 1 else None + k = 0 + for i in range(splat.positions.shape[0]): + xyz, rgb, opacity, scale, rot = _gaussian_item(splat, i, device) + if opacity_threshold > 0: + keep = opacity >= opacity_threshold + xyz, rgb, opacity, scale, rot = xyz[keep], rgb[keep], opacity[keep], scale[keep], rot[keep] + base_cam = camera_info + if base_cam is None: # no camera -> default 3/4 view, auto-framed on the splat + center = xyz.mean(0) if xyz.shape[0] else torch.zeros(3, device=device) + extent = (_quantile((xyz - center).norm(dim=-1), 0.99).clamp_min(1e-4) if xyz.shape[0] + else torch.tensor(1.0, device=device)) + dist = float(extent / (math.tan(math.radians(35.0) / 2) * 0.9)) + base_cam = _orbit_camera_info(35.0, 30.0, dist, 35.0, center, device) + for fr in range(n_frames): + cam_fr = (base_cam if n_frames == 1 + else _orbit_camera_info_yaw(base_cam, orbit_dir * 360.0 * fr / n_frames, device)) + bg_k = bg_imgs[k % bg_imgs.shape[0]] if bg_imgs is not None else bg # per-frame plate, or solid colour + img, mask = _render_gaussian(xyz, rgb, opacity, scale, rot, width, height, splat_scale, bg_k, cam_fr, + sharpen=sharpen, headlight_shading=headlight_shading, + render_style=render_style) + imgs.append(img) + masks.append(mask) + k += 1 + if pbar is not None: + pbar.update(1) + return IO.NodeOutput(torch.stack(imgs), torch.stack(masks)) + + +class CreateCameraInfo(IO.ComfyNode): # TODO: move to better file + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="CreateCameraInfo", + display_name="Create Camera Info", + search_aliases=["camera position", "make camera info", "orbit camera", "look at camera"], + category="3d", + description="Build a camera_info" + "Mode 'orbit' aims with yaw/pitch/distance around the target; " + "'look_at' places the camera at world position. Coordinates are the viewer's world space (right-handed,Y-up).", + inputs=[ + IO.DynamicCombo.Input("mode", options=[ + IO.DynamicCombo.Option("orbit", [ + IO.Float.Input("yaw", default=35.0, min=-360.0, max=360.0, step=1.0), + IO.Float.Input("pitch", default=30.0, min=-89.0, max=89.0, step=1.0), + IO.Float.Input("distance", default=4.0, min=0.01, max=1000.0, step=0.01, + tooltip="Camera distance from the target."), + ]), + IO.DynamicCombo.Option("look_at", [ + IO.Float.Input("position_x", default=4.0, min=-1000.0, max=1000.0, step=0.01, + tooltip="Camera position in world space (right-handed, Y-up)."), + IO.Float.Input("position_y", default=4.0, min=-1000.0, max=1000.0, step=0.01), + IO.Float.Input("position_z", default=4.0, min=-1000.0, max=1000.0, step=0.01), + ]), + IO.DynamicCombo.Option("quaternion", [ + IO.Float.Input("position_x", default=4.0, min=-1000.0, max=1000.0, step=0.01, + tooltip="Camera position in world space (right-handed, Y-up)."), + IO.Float.Input("position_y", default=4.0, min=-1000.0, max=1000.0, step=0.01), + IO.Float.Input("position_z", default=4.0, min=-1000.0, max=1000.0, step=0.01), + IO.Float.Input("quat_x", default=0.0, min=-1.0, max=1.0, step=0.001), + IO.Float.Input("quat_y", default=0.0, min=-1.0, max=1.0, step=0.001), + IO.Float.Input("quat_z", default=0.0, min=-1.0, max=1.0, step=0.001), + IO.Float.Input("quat_w", default=1.0, min=-1.0, max=1.0, step=0.001, + tooltip="Camera world-rotation quaternion (three.js: looks down local -Z). Normalized for you."), + ]), + ], tooltip="How to define the camera: orbit angles, an explicit position, or a position + quaternion."), + IO.Float.Input("target_x", default=0.0, min=-1000.0, max=1000.0, step=0.01, advanced=True, + tooltip="Look-at point (orbit pivot / aim). In orbit mode, move it to pan/translate the " + "whole camera. Ignored in quaternion mode. Defaults to the origin."), + IO.Float.Input("target_y", default=0.0, min=-1000.0, max=1000.0, step=0.01, advanced=True), + IO.Float.Input("target_z", default=0.0, min=-1000.0, max=1000.0, step=0.01, advanced=True), + IO.Float.Input("roll", default=0.0, min=-180.0, max=180.0, step=1.0, + tooltip="Camera roll about the view axis, degrees."), + IO.Float.Input("fov", default=35.0, min=1.0, max=120.0, step=1.0, + tooltip="Vertical field of view in degrees."), + IO.Float.Input("zoom", default=1.0, min=0.01, max=100.0, step=0.01, + tooltip="Digital zoom (focal-length multiplier). >1 zooms in without moving the camera."), + IO.Combo.Input("camera_type", options=["perspective", "orthographic"], + tooltip="Projection used by Render Splat: perspective (foreshortening) or orthographic (parallel)."), + ], + outputs=[IO.Load3DCamera.Output(display_name="camera_info")], + ) + + @classmethod + def execute(cls, mode, target_x, target_y, target_z, roll, fov, zoom=1.0, camera_type="perspective") -> IO.NodeOutput: + dev = comfy.model_management.get_torch_device() + kind = mode["mode"] + if kind == "quaternion": # explicit world position + camera rotation + position = [mode["position_x"], mode["position_y"], mode["position_z"]] + quat = [mode["quat_x"], mode["quat_y"], mode["quat_z"], mode["quat_w"]] + return IO.NodeOutput(_quat_camera_info(position, quat, fov, dev, zoom=zoom, camera_type=camera_type)) + target = [target_x, target_y, target_z] # orbit pivot / aim; move it to pan the whole camera + if kind == "orbit": # yaw/pitch/distance about the target (world Y-up) + y, p = math.radians(mode["yaw"]), math.radians(mode["pitch"]) + cy, sy, cp, sp = math.cos(y), math.sin(y), math.cos(p), math.sin(p) + d = mode["distance"] + position = [target_x + d * cp * sy, target_y + d * sp, target_z + d * cp * cy] + else: # look_at: explicit world-space camera position + position = [mode["position_x"], mode["position_y"], mode["position_z"]] + return IO.NodeOutput(_lookat_camera_info(position, target, fov, dev, zoom=zoom, camera_type=camera_type, roll=roll)) + + +class TransformSplat(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="TransformSplat", + display_name="Transform Splat", + search_aliases=["move splat", "rotate splat", "scale splat", "gaussian transform"], + category="3d/splat", + description="Translate, rotate, and scale a gaussian splat. " + "Non-uniform scale also reshapes every individual splat, slower process.", + inputs=[ + IO.Splat.Input("splat"), + IO.Float.Input("translate_x", default=0.0, min=-100.0, max=100.0, step=0.01), + IO.Float.Input("translate_y", default=0.0, min=-100.0, max=100.0, step=0.01), + IO.Float.Input("translate_z", default=0.0, min=-100.0, max=100.0, step=0.01), + IO.Float.Input("rotate_x", default=0.0, min=-360.0, max=360.0, step=1.0), + IO.Float.Input("rotate_y", default=0.0, min=-360.0, max=360.0, step=1.0), + IO.Float.Input("rotate_z", default=0.0, min=-360.0, max=360.0, step=1.0), + IO.Float.Input("scale_x", default=1.0, min=0.01, max=100.0, step=0.01), + IO.Float.Input("scale_y", default=1.0, min=0.01, max=100.0, step=0.01), + IO.Float.Input("scale_z", default=1.0, min=0.01, max=100.0, step=0.01), + ], + outputs=[IO.Splat.Output(display_name="splat")], + ) + + @classmethod + def execute(cls, splat, translate_x, translate_y, translate_z, + rotate_x, rotate_y, rotate_z, scale_x, scale_y, scale_z) -> IO.NodeOutput: + pos = splat.positions + dev, dt = pos.device, pos.dtype + q_rot = _euler_to_quat(rotate_x, rotate_y, rotate_z).to(device=dev, dtype=dt) + R = _quat_to_mat(q_rot[None])[0] # (3, 3) node rotation + D = torch.tensor([scale_x, scale_y, scale_z], dtype=dt, device=dev) + A = D[:, None] * R # diag(D) @ R: per-axis scale after rotation + t = torch.tensor([translate_x, translate_y, translate_z], dtype=dt, device=dev) + + positions = pos @ A.T + t # rotate, scale per-axis, then translate + if scale_x == scale_y == scale_z: # uniform: rotation/scale factor out cleanly + scales = splat.scales * scale_x + rotations = _quat_mul(q_rot.expand_as(splat.rotations), splat.rotations) + rotations = rotations / rotations.norm(dim=-1, keepdim=True).clamp_min(1e-12) + else: # non-uniform: transform Sigma = A R s^2 R^T A^T, re-extract + rg = _quat_to_mat(splat.rotations.reshape(-1, 4)) # (M,3,3) per-splat rotation + s2 = splat.scales.reshape(-1, 3).square() + cov = (rg * s2[:, None, :]) @ rg.transpose(-1, -2) # Sigma + cov = A @ cov @ A.T # A Sigma A^T (A broadcast over splats) + lam, V = torch.linalg.eigh(cov) # symmetric -> eigenvalues (asc), orthonormal axes + V = V * torch.where(torch.linalg.det(V) < 0, -1.0, 1.0)[..., None, None] # keep a proper rotation + scales = lam.clamp_min(0).sqrt().reshape(splat.scales.shape) + rotations = _mat_to_quat(V).reshape(splat.rotations.shape) + out = Types.SPLAT(positions, scales, rotations, splat.opacities, splat.sh, + counts=getattr(splat, "counts", None)) + return IO.NodeOutput(out) + + +class GetSplatCount(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="GetSplatCount", + display_name="Get Splat Count", + search_aliases=["splat count", "gaussian count", "number of splats", "splat info"], + category="3d/splat", + description="Returns the number of splats summed across the batch.", + inputs=[IO.Splat.Input("splat")], + outputs=[IO.Splat.Output(display_name="splat"), + IO.Int.Output(display_name="count"), + ], + hidden=[IO.Hidden.unique_id], + ) + + @classmethod + def execute(cls, splat) -> IO.NodeOutput: + count = sum(_real_len(splat, i) for i in range(splat.positions.shape[0])) + if cls.hidden.unique_id: # show the count inline on the node + PromptServer.instance.send_progress_text(f"{count:,} splats", cls.hidden.unique_id) + return IO.NodeOutput(splat, count) + + +def _pad_stack(items, n): + # Stack a list of (Lᵢ, *tail) tensors into (B, n, *tail), zero-padding each row up to n. + tail = items[0].shape[1:] + out = items[0].new_zeros((len(items), n, *tail)) + for i, t in enumerate(items): + out[i, :t.shape[0]] = t + return out + + +def _merge_gaussians(gaussians: list) -> Types.SPLAT: + # Concatenate SPLAT batches along the splat dimension (per item), padding SH to the highest degree. + gs = [g for g in gaussians if g is not None] + if not gs: + raise ValueError("MergeSplat: no gaussians to merge") + b = gs[0].positions.shape[0] + for g in gs: + if g.positions.shape[0] != b: + raise ValueError(f"MergeSplat: batch size mismatch ({b} vs {g.positions.shape[0]}).") + max_k = max(g.sh.shape[2] for g in gs) + + pos_b, scl_b, rot_b, op_b, sh_b, lengths = [], [], [], [], [], [] + for i in range(b): + pos_i, scl_i, rot_i, op_i, sh_i = [], [], [], [], [] + for g in gs: + end = _real_len(g, i) + pos_i.append(g.positions[i, :end]) + scl_i.append(g.scales[i, :end]) + rot_i.append(g.rotations[i, :end]) + op_i.append(g.opacities[i, :end]) + sh = g.sh[i, :end] # (end, K, 3) + if sh.shape[1] < max_k: # zero-pad lower-degree SH + sh = torch.cat([sh, sh.new_zeros(sh.shape[0], max_k - sh.shape[1], sh.shape[2])], dim=1) + sh_i.append(sh) + pos_b.append(torch.cat(pos_i)) + scl_b.append(torch.cat(scl_i)) + rot_b.append(torch.cat(rot_i)) + op_b.append(torch.cat(op_i)) + sh_b.append(torch.cat(sh_i)) + lengths.append(pos_b[-1].shape[0]) + + n = max(lengths) + counts = None + if len(set(lengths)) > 1: + counts = torch.tensor(lengths, device=gs[0].positions.device, dtype=torch.int64) + return Types.SPLAT(_pad_stack(pos_b, n), _pad_stack(scl_b, n), _pad_stack(rot_b, n), + _pad_stack(op_b, n), _pad_stack(sh_b, n), counts=counts) + + +class MergeSplat(IO.ComfyNode): + @classmethod + def define_schema(cls): + # Autogrow: a splat0/splat1/... input list that grows a fresh slot as you connect splats. + splats = IO.Autogrow.TemplatePrefix(IO.Splat.Input("splat"), prefix="splat", min=2, max=32) + return IO.Schema( + node_id="MergeSplat", + display_name="Merge Splats", + search_aliases=["union splat", "densify gaussian", "combine splat", "merge gaussian"], + category="3d/splat", + description="Concatenate any number of gaussian splats into one. Unioning several decodes of the same " + "latent at different seeds densifies the surface, this can improve surface quality when meshing.", + inputs=[IO.Autogrow.Input("splats", template=splats)], + outputs=[IO.Splat.Output(display_name="splat")], + ) + + @classmethod + def execute(cls, splats: IO.Autogrow.Type) -> IO.NodeOutput: + gs = [v for v in splats.values() if v is not None] + if not gs: + raise ValueError("MergeSplat: connect at least one splat.") + return IO.NodeOutput(_merge_gaussians(gs)) + + +def _inverse_covariance(scale, quat): + # Per-splat Sigma^-1 = R diag(1/s^2) R^T. scale (N,3) linear std, quat (N,4) wxyz -> (N,3,3). + q = quat / quat.norm(dim=1, keepdim=True).clamp_min(1e-12) + w, x, y, z = q.unbind(-1) + R = torch.stack([ + 1 - 2 * (y * y + z * z), 2 * (x * y - w * z), 2 * (x * z + w * y), + 2 * (x * y + w * z), 1 - 2 * (x * x + z * z), 2 * (y * z - w * x), + 2 * (x * z - w * y), 2 * (y * z + w * x), 1 - 2 * (x * x + y * y), + ], dim=1).reshape(-1, 3, 3) + inv_s2 = 1.0 / scale.clamp_min(1e-8) ** 2 # (N, 3) + return torch.einsum("nij,nj,nkj->nik", R, inv_s2, R) + + +def _splat_density(xyz, opacity, scale, quat, rgb, res, kernel, device, color_sharpen=1.0, chunk=4096, progress=None, + col_dtype=torch.float16): + # Splat each gaussian as its oriented-covariance disk (3-sigma, opacity-weighted) into a density grid, + # plus a colour volume. Each gaussian uses a voxel window sized to its OWN 3-sigma (capped at `kernel`). + # Colour is weighted by w^color_sharpen: >1 biases each voxel toward its dominant gaussian (crisper + # texture). Returns (density, colour numerator, colour normaliser, origin, voxel). + pad = 4.0 * scale.median() + lo = xyz.amin(0) - pad + hi = xyz.amax(0) + pad + voxel = ((hi - lo).max() / res).clamp_min(1e-8) + dx, dy, dz = (torch.ceil((hi - lo) / voxel).long() + 1).tolist() + + sinv = _inverse_covariance(scale, quat) + kreq = torch.ceil(3.0 * scale.amax(-1) / voxel).long().clamp(1, int(kernel)) # per-gaussian half-width + sharp = color_sharpen != 1.0 + vol = torch.zeros(dx * dy * dz, device=device) # Sum(w) density (surface) + colvol = torch.zeros(dx * dy * dz, 3, device=device, dtype=col_dtype) # Sum(w^p * rgb) colour numerator + wcol = torch.zeros(dx * dy * dz, device=device, dtype=col_dtype) if sharp else None # Sum(w^p) normaliser (p>1) + n, done = xyz.shape[0], 0 + for k in range(1, int(kernel) + 1): + sel = (kreq == k).nonzero(as_tuple=True)[0] + if sel.numel() == 0: + continue + rng = torch.arange(-k, k + 1, device=device, dtype=torch.float32) + off = torch.stack(torch.meshgrid(rng, rng, rng, indexing="ij"), -1).reshape(-1, 3) # (M, 3) + for st in range(0, sel.numel(), chunk): + gi = sel[st:st + chunk] + cc = xyz[gi] + idx = ((cc - lo) / voxel).round()[:, None, :] + off[None] # (b, M, 3) voxel coords + d = (lo + idx * voxel) - cc[:, None, :] # world offset to voxel center + quad = torch.einsum("bmi,bij,bmj->bm", d, sinv[gi], d) + wgt = opacity[gi, None] * torch.exp(-0.5 * quad) + wgt = torch.where(quad < 9.0, wgt, torch.zeros_like(wgt)) # clip beyond 3 sigma + ii = idx.long() + ix = ii[..., 0].clamp(0, dx - 1) + iy = ii[..., 1].clamp(0, dy - 1) + iz = ii[..., 2].clamp(0, dz - 1) + flat = (ix * (dy * dz) + iy * dz + iz).reshape(-1) + vol.index_add_(0, flat, wgt.reshape(-1)) + wp = wgt.pow(color_sharpen) if sharp else wgt # winner-take-more colour weight + colvol.index_add_(0, flat, (wp[..., None] * rgb[gi, None, :]).reshape(-1, 3).to(col_dtype)) + if sharp: + wcol.index_add_(0, flat, wp.reshape(-1).to(col_dtype)) + done += gi.numel() + if progress is not None: + progress(min(1.0, done / max(1, n))) + colnorm = (wcol if sharp else vol).reshape(dx, dy, dz) # p==1 -> Sum(w) == density + return vol.reshape(dx, dy, dz), colvol.reshape(dx, dy, dz, 3), colnorm, lo.cpu().numpy(), float(voxel) + + +def _connected_components_gpu(faces, nv): + # FastSV connected components: grandparent hooking + shortcutting, ~O(log nv) iterations. + # Returns per-vertex component labels (min node id, not densified). + a = torch.cat([faces[:, 0], faces[:, 1]]) # 2F edge endpoints: (v0,v1),(v1,v2) + b = torch.cat([faces[:, 1], faces[:, 2]]) + f = torch.arange(nv, device=faces.device) + while True: + gp = f[f] # grandparent + ga, gb = gp[a], gp[b] + new = f.clone() + new.scatter_reduce_(0, f[a], gb, "amin", include_self=True) # stochastic hooking onto roots + new.scatter_reduce_(0, f[b], ga, "amin", include_self=True) + new.scatter_reduce_(0, a, gb, "amin", include_self=True) # aggressive hooking, both directions + new.scatter_reduce_(0, b, ga, "amin", include_self=True) + new = new[new] # shortcut (path compression) + if torch.equal(new, f): + return f + f = new + + +def _clean_components_gpu(verts, faces, min_verts, device): + # GPU port of _clean_components: FastSV components + scatter reductions. Byte-identical to the numpy path + vt = torch.as_tensor(verts, device=device) + ft = torch.as_tensor(faces, device=device) + nv = vt.shape[0] + _, label = torch.unique(_connected_components_gpu(ft, nv), return_inverse=True) # dense 0..ncomp-1 + ncomp = int(label.max()) + 1 + flabel = label[ft[:, 0]] # component id per face + keep = torch.bincount(label, minlength=ncomp) >= min_verts # per-component vertex-count gate + if int(keep.sum()) > 1: + fcount = torch.bincount(flabel, minlength=ncomp) + largest = int(torch.where(keep, fcount, fcount.new_tensor(-1)).argmax()) + v0, v1, v2 = vt[ft[:, 0]], vt[ft[:, 1]], vt[ft[:, 2]] + cvol = torch.zeros(ncomp, device=device).scatter_add_(0, flabel, (v0 * torch.linalg.cross(v1, v2)).sum(-1)) + idx3 = label[:, None].expand(-1, 3) # per-component vertex bbox + cmin = torch.full((ncomp, 3), float("inf"), device=device).scatter_reduce_(0, idx3, vt, "amin", include_self=True) + cmax = torch.full((ncomp, 3), float("-inf"), device=device).scatter_reduce_(0, idx3, vt, "amax", include_self=True) + tol = 1e-4 * (cmax[largest] - cmin[largest]).max() + enclosed = (cmin >= cmin[largest] - tol).all(1) & (cmax <= cmax[largest] + tol).all(1) + inner = enclosed & (torch.sign(cvol) != torch.sign(cvol[largest])) & (torch.arange(ncomp, device=device) != largest) + keep &= ~inner + faces_k = ft[keep[flabel]] + if faces_k.shape[0] == 0: + return verts[:0], faces[:0] + used = torch.unique(faces_k) # sorted, matches np.unique + remap = torch.full((nv,), -1, dtype=torch.int64, device=device) + remap[used] = torch.arange(used.shape[0], device=device) + return vt[used].cpu().numpy(), remap[faces_k].cpu().numpy() + + +def _clean_components(verts, faces, min_verts, device=None): + # Drop floaters (components with < min_verts vertices) and inner shells - the surfel shell density + # extracts a double wall (outer + inner cavity surface). GPU path (FastSV CC + scatter reductions, ~13x + # faster) when an accelerator has headroom; else numpy/scipy. Both produce byte-identical output. + if device is not None and not comfy.model_management.is_device_cpu(device) and \ + comfy.model_management.get_free_memory(device) > 10 * faces.size * 8: # peak ~8.4x faces bytes + return _clean_components_gpu(verts, faces, min_verts, device) + nv = len(verts) + e = np.concatenate([faces[:, [0, 1]], faces[:, [1, 2]], faces[:, [0, 2]]], 0) + ncomp, label = connected_components(coo_matrix((np.ones(len(e)), (e[:, 0], e[:, 1])), shape=(nv, nv)), directed=False) + flabel = label[faces[:, 0]] # component id per face + keep = np.bincount(label, minlength=ncomp) >= min_verts # per-component vertex-count gate + if keep.sum() > 1: + fcount = np.bincount(flabel, minlength=ncomp) + largest = np.where(keep, fcount, -1).argmax() + v0, v1, v2 = verts[faces[:, 0]], verts[faces[:, 1]], verts[faces[:, 2]] + cvol = np.bincount(flabel, weights=np.einsum("ij,ij->i", v0, np.cross(v1, v2)), minlength=ncomp) # 6*signed vol + cidx = np.arange(ncomp) # per-component vertex bbox via ndimage (~6x faster than ufunc.at) + cmin = np.stack([_ndi_minimum(verts[:, a], label, cidx) for a in range(3)], 1) + cmax = np.stack([_ndi_maximum(verts[:, a], label, cidx) for a in range(3)], 1) + tol = 1e-4 * (cmax[largest] - cmin[largest]).max() + enclosed = (cmin >= cmin[largest] - tol).all(1) & (cmax <= cmax[largest] + tol).all(1) + inner = enclosed & (np.sign(cvol) != np.sign(cvol[largest])) & (np.arange(ncomp) != largest) + keep &= ~inner + faces = faces[keep[flabel]] + if len(faces) == 0: + return verts[:0], faces + used = np.unique(faces) + remap = np.full(nv, -1, np.int64) + remap[used] = np.arange(len(used)) + return verts[used], remap[faces] + + +def _surface_nets(vol, level, voxel, origin, device): + # Vectorized Surface Nets: one dual vertex per sign-changing cell at its edge-crossing mean, quads wound CCW-outward. + # Returns verts (V,3), faces (F,3). + vol = vol.to(device=device, dtype=torch.float32) + dx, dy, dz = vol.shape + origin_t = torch.as_tensor(origin, device=device, dtype=torch.float32) + empty = (np.zeros((0, 3), np.float32), np.zeros((0, 3), np.int64)) + if dx < 2 or dy < 2 or dz < 2: + return empty + + # Active = cells whose 8 corners aren't all in/all out. + inside = vol >= level # (dx,dy,dz) bool + cs8 = [inside[ox:ox + dx - 1, oy:oy + dy - 1, oz:oz + dz - 1] + for ox, oy, oz in ((0, 0, 0), (1, 0, 0), (0, 1, 0), (1, 1, 0), + (0, 0, 1), (1, 0, 1), (0, 1, 1), (1, 1, 1))] + any_in = cs8[0] | cs8[1] | cs8[2] | cs8[3] | cs8[4] | cs8[5] | cs8[6] | cs8[7] + all_in = cs8[0] & cs8[1] & cs8[2] & cs8[3] & cs8[4] & cs8[5] & cs8[6] & cs8[7] + active = any_in & ~all_in # (cx,cy,cz) straddling cells + nv = int(active.sum()) + if nv == 0: + return empty + + # Active cells only (a thin shell): each dual vertex = mean of its 12 edges' zero-crossings. + del any_in, all_in, cs8 # corner bool grids no longer needed + ac = active.nonzero(as_tuple=False) # (nv,3) cell min-corner indices + offs = torch.tensor([[0, 0, 0], [1, 0, 0], [0, 1, 0], [1, 1, 0], + [0, 0, 1], [1, 0, 1], [0, 1, 1], [1, 1, 1]], device=device) + offf = offs.to(torch.float32) + edges = torch.tensor([[0, 1], [0, 2], [0, 4], [1, 3], [1, 5], [2, 3], + [2, 6], [3, 7], [4, 5], [4, 6], [5, 7], [6, 7]], device=device) + e0, e1 = edges[:, 0], edges[:, 1] + oe0, oe1 = offf[e0], offf[e1] # (12,3) edge endpoints + + cstep = 1 << 18 # chunk to bound peak memory (CPU RAM too) + loc = [] + for st in range(0, nv, cstep): + ci = ac[st:st + cstep, None, :] + offs[None] # (m,8,3) + cval = vol[ci[..., 0], ci[..., 1], ci[..., 2]] # (m,8) corner values + csl = cval >= level + v0, v1 = cval[:, e0], cval[:, e1] # (m,12) + cross = (csl[:, e0] != csl[:, e1])[..., None].to(torch.float32) + denom = v1 - v0 + t = torch.where(denom.abs() > 1e-12, (level - v0) / denom, torch.full_like(denom, 0.5)).clamp(0, 1) + pts = torch.lerp(oe0, oe1, t[..., None]) # (m,12,3) local crossings (fused interp) + loc.append((pts * cross).sum(1) / cross.sum(1).clamp_min(1.0)) # (m,3) in [0,1] + local = torch.cat(loc, 0) if len(loc) > 1 else loc[0] # (nv,3) + verts = origin_t + (ac.to(torch.float32) + local) * voxel # world space + del loc, local, ac + + vid = torch.full((dx - 1, dy - 1, dz - 1), -1, dtype=torch.int32, device=device) + vid[active] = torch.arange(nv, dtype=torch.int32, device=device) + del active + + # Each straddling grid edge -> one quad from its 4 cells; `sol` (low-end sign) picks outward winding. + faces = [] + + def emit(cr, sol, a, b, d, c): + valid = cr & (a >= 0) & (b >= 0) & (c >= 0) & (d >= 0) + if not bool(valid.any()): + return + a, b, c, d, sol = a[valid], b[valid], c[valid], d[valid], sol[valid] + p2, p4 = torch.where(sol, b, c), torch.where(sol, c, b) # reverse quad winding where ~sol + faces.append(torch.stack([a, p2, d], 1)) + faces.append(torch.stack([a, d, p4], 1)) + + a = inside[0:dx - 1, 1:dy - 1, 1:dz - 1] + emit(a != inside[1:dx, 1:dy - 1, 1:dz - 1], a, + vid[:, 0:dy - 2, 0:dz - 2], vid[:, 1:dy - 1, 0:dz - 2], + vid[:, 1:dy - 1, 1:dz - 1], vid[:, 0:dy - 2, 1:dz - 1]) + a = inside[1:dx - 1, 0:dy - 1, 1:dz - 1] + emit(a != inside[1:dx - 1, 1:dy, 1:dz - 1], a, + vid[0:dx - 2, :, 0:dz - 2], vid[0:dx - 2, :, 1:dz - 1], + vid[1:dx - 1, :, 1:dz - 1], vid[1:dx - 1, :, 0:dz - 2]) + a = inside[1:dx - 1, 1:dy - 1, 0:dz - 1] + emit(a != inside[1:dx - 1, 1:dy - 1, 1:dz], a, + vid[0:dx - 2, 0:dy - 2, :], vid[1:dx - 1, 0:dy - 2, :], + vid[1:dx - 1, 1:dy - 1, :], vid[0:dx - 2, 1:dy - 1, :]) + + if not faces: + return empty + return verts.cpu().numpy().astype(np.float32), torch.cat(faces, 0).cpu().numpy().astype(np.int64) + + +def _otsu_level(values, bins=256): + # Otsu threshold: the density value that best splits inside/outside (max between-class variance). + hist, edges = np.histogram(values, bins=bins) + hist = hist.astype(np.float64) + centers = (edges[:-1] + edges[1:]) * 0.5 + w = np.cumsum(hist) # background-class weight at each split + mu = np.cumsum(hist * centers) + wf = w[-1] - w # foreground-class weight + mb = mu / np.where(w > 0, w, 1.0) + mf = (mu[-1] - mu) / np.where(wf > 0, wf, 1.0) + var_b = w * wf * (mb - mf) ** 2 # between-class variance + var_b[(w <= 0) | (wf <= 0)] = -1.0 + return float(centers[int(np.argmax(var_b))]) + + +def _taubin_smooth(verts, faces, iters, lam=0.5, mu=-0.53): + # Taubin lambda|mu smoothing: low-pass the mesh surface without the shrinkage of a Laplacian blur + # (the mu inflation pass cancels the lambda pass's volume loss). Uniform (umbrella) weights. + if iters <= 0 or len(verts) == 0 or len(faces) == 0: + return verts + nv = len(verts) + e = np.concatenate([faces[:, [0, 1]], faces[:, [1, 2]], faces[:, [0, 2]]], 0) + e = np.concatenate([e, e[:, ::-1]], 0) # symmetric adjacency + adj = coo_matrix((np.ones(len(e), np.float32), (e[:, 0], e[:, 1])), shape=(nv, nv)).tocsr() + adj.data[:] = 1.0 + deg = np.clip(np.asarray(adj.sum(1)).ravel(), 1.0, None).astype(np.float32)[:, None] + v = verts.astype(np.float32) # fp32 matvec: ~2x faster, sub-micron drift on unit-scale verts + for _ in range(int(iters)): + for fac in (lam, mu): + v = v + np.float32(fac) * ((adj @ v) / deg - v) # fac * (mean(neighbours) - v) + return np.ascontiguousarray(v) + + +def _sample_vertex_colours_gpu(colvol, colnorm, verts, origin, voxel, device): + # GPU trilinear sampling of the colour numerator (3ch) and normaliser (1ch) at vertex grid-coords + # reproduces scipy map_coordinates(order=1, mode='nearest'). Returns col (V,3) numpy. + dx, dy, dz = colnorm.shape + vt = torch.as_tensor(verts, device=device, dtype=torch.float32) + org = torch.as_tensor(origin, device=device, dtype=torch.float32) + gi = (vt - org) / voxel # (V,3) grid-index coords (x,y,z) + size = torch.tensor([dx, dy, dz], device=device, dtype=torch.float32) + g = 2.0 * gi / (size - 1).clamp_min(1.0) - 1.0 # -> [-1,1] (align_corners) + grid = torch.stack([g[:, 2], g[:, 1], g[:, 0]], -1)[None, None, None] # (1,1,1,V,3): grid_sample order (W=z,H=y,D=x) + + def samp(v): # (dx,dy,dz,C) cpu fp16 -> (C,V) fp32 on device + inp = v.to(device).permute(3, 0, 1, 2)[None].float() + o = torch.nn.functional.grid_sample(inp, grid, mode="bilinear", padding_mode="border", align_corners=True) + return o[0, :, 0, 0, :] + num = samp(colvol) # (3,V) + den = samp(colnorm[..., None]) # (1,V) + return (num / den.clamp_min(1e-8)).T.cpu().numpy() # (V,3) + + +def _gaussian_to_mesh(g: Types.SPLAT, i, res, kernel, taubin, level_bias, min_component, min_opacity, color_sharpen, device, progress=None): + # Mesh one splat: density + colour grids -> Surface Nets -> floater removal -> Taubin smoothing -> + # volume-sampled colours. Returns (verts, faces int64, colors in [0,1]), or None if no surface. + rep = progress if progress is not None else (lambda *_: None) + + end = _real_len(g, i) + xyz = g.positions[i, :end].to(device=device, dtype=torch.float32) + scale = g.scales[i, :end].to(device=device, dtype=torch.float32) + quat = g.rotations[i, :end].to(device=device, dtype=torch.float32) + opacity = g.opacities[i, :end].reshape(-1).to(device=device, dtype=torch.float32) + rgb = (g.sh[i, :end, 0, :].to(device=device, dtype=torch.float32) * _C0 + 0.5).clamp(0, 1) + + keep = opacity >= min_opacity + xyz, scale, quat, opacity, rgb = xyz[keep], scale[keep], quat[keep], opacity[keep], rgb[keep] + if xyz.shape[0] == 0: + return None + + vol, colvol, colnorm, origin, voxel = _splat_density(xyz, opacity, scale, quat, rgb, res, kernel, device, + color_sharpen=color_sharpen, + progress=lambda f: rep(0.25 * f)) # density build: 0 -> 25% + # Colour: sample on the GPU (grid_sample) when there's headroom + colour_gpu = not comfy.model_management.is_device_cpu(device) and comfy.model_management.get_free_memory(device) > 6 * vol.numel() * 4 + if colour_gpu: + colvol_cpu, colnorm_cpu = colvol.cpu(), colnorm.half().cpu() # park colours (fp16) off-GPU during meshing + colvol_np = colnorm_np = None + else: + colvol_np = colvol.cpu().numpy().astype(np.float32) # Sum(w^p * rgb) colour numerator (fp16 grid -> fp32) + colnorm_np = colnorm.cpu().numpy().astype(np.float32) # Sum(w^p) colour normaliser + del colvol, colnorm # free the colour grids before iso-surfacing + rep(0.40) + + vmin, vmax = float(vol.min()), float(vol.max()) + occ = vol[vol > vmax * 1e-3] # occupied voxels (skip the empty-space peak) + if occ.numel() == 0: + return None + # Otsu picks the inside/outside split principledly; `level_bias` nudges it (1.0 = auto). Clamp strictly + # inside the data range so a bias can't push the iso off the histogram. + level = min(max(_otsu_level(occ.cpu().numpy()) * level_bias, vmin + 1e-6 * (vmax - vmin)), + vmax - 1e-6 * (vmax - vmin)) + + # Iso-surface on the accelerator when there's headroom: ~15x faster than CPU, identical output. Chunked + # Surface Nets peaks at ~3-3.5x the density grid, so fall back to CPU for large grids / tight VRAM. + sn_dev = device + if not comfy.model_management.is_device_cpu(device) and comfy.model_management.get_free_memory(device) < 6 * vol.numel() * 4: + sn_dev = torch.device("cpu") + vol = vol.cpu() + verts, faces = _surface_nets(vol, level, voxel, origin, sn_dev) + del vol + rep(0.55) + if min_component > 0 and len(faces) > 0: + verts, faces = _clean_components(verts, faces, min_component, device) + if len(verts) == 0 or len(faces) == 0: + return None + + # Taubin smooths the blocky iso without shrinking it (unlike blurring the density, which rounds features). + verts = _taubin_smooth(verts, faces, taubin) + rep(0.7) + + # Colour each vertex from the co-splatted colour volume: trilinearly sample the numerator Sum(w^p*rgb) + # and normaliser Sum(w^p) separately, then divide. Normalising AFTER interpolation keeps zero-density + # edge voxels from pulling colours toward black, and matches the gaussians that formed the surface. + if colour_gpu: + col = _sample_vertex_colours_gpu(colvol_cpu, colnorm_cpu, verts, origin, voxel, device) + else: + coords = ((verts - origin) / voxel).T # (3, V) grid-index coords, matching volume axes + num = np.stack([map_coordinates(colvol_np[..., c], coords, order=1, mode="nearest") for c in range(3)], -1) + den = map_coordinates(colnorm_np, coords, order=1, mode="nearest") + col = num / np.clip(den, 1e-8, None)[:, None] + rep(1.0) + + # The unlit material's COLOR_0 is linear and the viewer sRGB-encodes it on output; the splat colours + # are display (sRGB) values, so convert sRGB -> linear here to land at the same brightness as the splat. + col = np.clip(col, 0, 1) + col = np.where(col <= 0.04045, col / 12.92, ((col + 0.055) / 1.055) ** 2.4).astype(np.float32) + + # Splat +Y is glTF's -Y: rotate 180 deg about X (negate Y,Z) to land upright. Proper rotation, so + # winding is kept; done after colouring (which works in the splat frame). + verts = np.ascontiguousarray(verts * np.array([1.0, -1.0, -1.0], dtype=np.float32)) + return (torch.from_numpy(verts), torch.from_numpy(faces), torch.from_numpy(col)) + + +class SplatToMesh(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="SplatToMesh", + display_name="Extract Mesh from Splat", + search_aliases=["splat to mesh", "gaussian surface nets", "splat surface", "mesh splat"], + category="3d/splat", + description="Extract a coloured mesh from a gaussian splat.", + inputs=[ + IO.Splat.Input("splat"), + IO.Int.Input("resolution", default=384, min=64, max=768, step=16, + tooltip="Density-grid resolution along the longest axis. Higher = finer surface, " + "more VRAM/time (grows with resolution^3)."), + IO.Int.Input("kernel", default=5, min=1, max=8, + tooltip="Max splat half-width in voxels. Each gaussian is rasterized over a window " + "sized to its own 3-sigma, capped here - small surfels stay cheap, large ones " + "aren't truncated. Raise if sparse splats leave gaps."), + IO.Int.Input("smooth", default=0, min=0, max=60, advanced = True, + tooltip="Taubin mesh-smoothing iterations. Smooths the surface without shrinking it " + "(volume-preserving), unlike blurring the density. 0 = raw surface."), + IO.Float.Input("level", default=0.4, min=0.0, max=2.0, step=0.01, + tooltip="Iso-surface level. Auto-picked by Otsu; this biases it (1.0 = auto, lower = " + "fatter/more-connected surface, higher = thinner/tighter)."), + IO.Int.Input("min_component", default=500, min=0, max=100000, step=50, advanced=True, + tooltip="Drop connected components smaller than this many vertices (0 = keep all). " + "Removes detached floater blobs and the inner shell of the double wall."), + IO.Float.Input("min_opacity", default=0.02, min=0.0, max=1.0, step=0.01, advanced=True, + tooltip="Ignore gaussians fainter than this before meshing."), + IO.Float.Input("color_sharpen", default=2.0, min=1.0, max=8.0, step=0.5, + tooltip="Crisp up the vertex texture: 1.0 = physically-correct blend; higher biases " + "each voxel's colour toward its dominant gaussian instead of averaging " + "neighbours (de-smears the texture). Colour only - geometry is unchanged."), + ], + outputs=[IO.Mesh.Output(display_name="mesh")], + ) + + @classmethod + def execute(cls, splat, resolution, kernel, smooth, level, min_component, min_opacity, color_sharpen) -> IO.NodeOutput: + device = comfy.model_management.get_torch_device() + b = splat.positions.shape[0] + prec = 1000 # each splat owns a 0..prec block of the bar; its callback advances within that block + pbar = comfy.utils.ProgressBar(b * prec) + + verts_l, faces_l, colors_l = [], [], [] + for i in range(b): + cb = lambda f, base=i * prec: pbar.update_absolute(base + int(min(max(f, 0.0), 1.0) * prec)) + res = _gaussian_to_mesh(splat, i, resolution, kernel, smooth, level, min_component, min_opacity, color_sharpen, device, cb) + if res is None: + logging.warning("SplatToMesh: splat %d produced no surface; emitting an empty mesh.", i) + v, f, c = torch.zeros((0, 3)), torch.zeros((0, 3), dtype=torch.int64), torch.zeros((0, 3)) + else: + v, f, c = res + verts_l.append(v) + faces_l.append(f) + colors_l.append(c) + pbar.update_absolute((i + 1) * prec) # snap to block end (covers empty / early-out splats) + # unlit: render flat (emissive-like) so SaveGLB matches the splat instead of lighting/washing it. + return IO.NodeOutput(pack_variable_mesh_batch(verts_l, faces_l, colors=colors_l, unlit=True)) + + +class GaussianExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[IO.ComfyNode]]: + return [SplatToFile3D, File3DToSplat, RenderSplat, CreateCameraInfo, TransformSplat, + GetSplatCount, MergeSplat, SplatToMesh] + + +async def comfy_entrypoint() -> GaussianExtension: + return GaussianExtension() diff --git a/ComfyUI/comfy_extras/nodes_gits.py b/ComfyUI/comfy_extras/nodes_gits.py new file mode 100644 index 0000000000000000000000000000000000000000..434a24387b41085edd8aa6109292144528618871 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_gits.py @@ -0,0 +1,382 @@ +# from https://github.com/zju-pi/diff-sampler/tree/main/gits-main +import numpy as np +import torch +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io + +def loglinear_interp(t_steps, num_steps): + """ + Performs log-linear interpolation of a given array of decreasing numbers. + """ + xs = np.linspace(0, 1, len(t_steps)) + ys = np.log(t_steps[::-1]) + + new_xs = np.linspace(0, 1, num_steps) + new_ys = np.interp(new_xs, xs, ys) + + interped_ys = np.exp(new_ys)[::-1].copy() + return interped_ys + +NOISE_LEVELS = { + 0.80: [ + [14.61464119, 7.49001646, 0.02916753], + [14.61464119, 11.54541874, 6.77309084, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 3.07277966, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 5.85520077, 2.05039096, 0.02916753], + [14.61464119, 12.2308979, 8.75849152, 7.49001646, 5.85520077, 2.05039096, 0.02916753], + [14.61464119, 12.2308979, 8.75849152, 7.49001646, 5.85520077, 3.07277966, 1.56271636, 0.02916753], + [14.61464119, 12.96784878, 11.54541874, 8.75849152, 7.49001646, 5.85520077, 3.07277966, 1.56271636, 0.02916753], + [14.61464119, 13.76078796, 12.2308979, 10.90732002, 8.75849152, 7.49001646, 5.85520077, 3.07277966, 1.56271636, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 10.90732002, 8.75849152, 7.49001646, 5.85520077, 3.07277966, 1.56271636, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 10.90732002, 9.24142551, 8.30717278, 7.49001646, 5.85520077, 3.07277966, 1.56271636, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 10.90732002, 9.24142551, 8.30717278, 7.49001646, 6.14220476, 4.86714602, 3.07277966, 1.56271636, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 11.54541874, 10.31284904, 9.24142551, 8.30717278, 7.49001646, 6.14220476, 4.86714602, 3.07277966, 1.56271636, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 11.54541874, 10.90732002, 10.31284904, 9.24142551, 8.30717278, 7.49001646, 6.14220476, 4.86714602, 3.07277966, 1.56271636, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 11.54541874, 10.90732002, 10.31284904, 9.24142551, 8.75849152, 8.30717278, 7.49001646, 6.14220476, 4.86714602, 3.07277966, 1.56271636, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 11.54541874, 10.90732002, 10.31284904, 9.24142551, 8.75849152, 8.30717278, 7.49001646, 6.14220476, 4.86714602, 3.1956799, 1.98035145, 0.86115354, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 11.54541874, 10.90732002, 10.31284904, 9.75859547, 9.24142551, 8.75849152, 8.30717278, 7.49001646, 6.14220476, 4.86714602, 3.1956799, 1.98035145, 0.86115354, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 11.54541874, 10.90732002, 10.31284904, 9.75859547, 9.24142551, 8.75849152, 8.30717278, 7.49001646, 6.77309084, 5.85520077, 4.65472794, 3.07277966, 1.84880662, 0.83188516, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 11.54541874, 10.90732002, 10.31284904, 9.75859547, 9.24142551, 8.75849152, 8.30717278, 7.88507891, 7.49001646, 6.77309084, 5.85520077, 4.65472794, 3.07277966, 1.84880662, 0.83188516, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 11.54541874, 10.90732002, 10.31284904, 9.75859547, 9.24142551, 8.75849152, 8.30717278, 7.88507891, 7.49001646, 6.77309084, 5.85520077, 4.86714602, 3.75677586, 2.84484982, 1.78698075, 0.803307, 0.02916753], + ], + 0.85: [ + [14.61464119, 7.49001646, 0.02916753], + [14.61464119, 7.49001646, 1.84880662, 0.02916753], + [14.61464119, 11.54541874, 6.77309084, 1.56271636, 0.02916753], + [14.61464119, 11.54541874, 7.11996698, 3.07277966, 1.24153244, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 5.09240818, 2.84484982, 0.95350921, 0.02916753], + [14.61464119, 12.2308979, 8.75849152, 7.49001646, 5.09240818, 2.84484982, 0.95350921, 0.02916753], + [14.61464119, 12.2308979, 8.75849152, 7.49001646, 5.58536053, 3.1956799, 1.84880662, 0.803307, 0.02916753], + [14.61464119, 12.96784878, 11.54541874, 8.75849152, 7.49001646, 5.58536053, 3.1956799, 1.84880662, 0.803307, 0.02916753], + [14.61464119, 12.96784878, 11.54541874, 8.75849152, 7.49001646, 6.14220476, 4.65472794, 3.07277966, 1.84880662, 0.803307, 0.02916753], + [14.61464119, 13.76078796, 12.2308979, 10.90732002, 8.75849152, 7.49001646, 6.14220476, 4.65472794, 3.07277966, 1.84880662, 0.803307, 0.02916753], + [14.61464119, 13.76078796, 12.2308979, 10.90732002, 9.24142551, 8.30717278, 7.49001646, 6.14220476, 4.65472794, 3.07277966, 1.84880662, 0.803307, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 10.90732002, 9.24142551, 8.30717278, 7.49001646, 6.14220476, 4.65472794, 3.07277966, 1.84880662, 0.803307, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 11.54541874, 10.31284904, 9.24142551, 8.30717278, 7.49001646, 6.14220476, 4.65472794, 3.07277966, 1.84880662, 0.803307, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 11.54541874, 10.31284904, 9.24142551, 8.30717278, 7.49001646, 6.14220476, 4.86714602, 3.60512662, 2.6383388, 1.56271636, 0.72133851, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 11.54541874, 10.31284904, 9.24142551, 8.30717278, 7.49001646, 6.77309084, 5.85520077, 4.65472794, 3.46139455, 2.45070267, 1.56271636, 0.72133851, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 11.54541874, 10.31284904, 9.24142551, 8.75849152, 8.30717278, 7.49001646, 6.77309084, 5.85520077, 4.65472794, 3.46139455, 2.45070267, 1.56271636, 0.72133851, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 11.54541874, 10.90732002, 10.31284904, 9.24142551, 8.75849152, 8.30717278, 7.49001646, 6.77309084, 5.85520077, 4.65472794, 3.46139455, 2.45070267, 1.56271636, 0.72133851, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 11.54541874, 10.90732002, 10.31284904, 9.75859547, 9.24142551, 8.75849152, 8.30717278, 7.49001646, 6.77309084, 5.85520077, 4.65472794, 3.46139455, 2.45070267, 1.56271636, 0.72133851, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 11.54541874, 10.90732002, 10.31284904, 9.75859547, 9.24142551, 8.75849152, 8.30717278, 7.88507891, 7.49001646, 6.77309084, 5.85520077, 4.65472794, 3.46139455, 2.45070267, 1.56271636, 0.72133851, 0.02916753], + ], + 0.90: [ + [14.61464119, 6.77309084, 0.02916753], + [14.61464119, 7.49001646, 1.56271636, 0.02916753], + [14.61464119, 7.49001646, 3.07277966, 0.95350921, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 2.54230714, 0.89115214, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 4.86714602, 2.54230714, 0.89115214, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 5.09240818, 3.07277966, 1.61558151, 0.69515091, 0.02916753], + [14.61464119, 12.2308979, 8.75849152, 7.11996698, 4.86714602, 3.07277966, 1.61558151, 0.69515091, 0.02916753], + [14.61464119, 12.2308979, 8.75849152, 7.49001646, 5.85520077, 4.45427561, 2.95596409, 1.61558151, 0.69515091, 0.02916753], + [14.61464119, 12.2308979, 8.75849152, 7.49001646, 5.85520077, 4.45427561, 3.1956799, 2.19988537, 1.24153244, 0.57119018, 0.02916753], + [14.61464119, 12.96784878, 10.90732002, 8.75849152, 7.49001646, 5.85520077, 4.45427561, 3.1956799, 2.19988537, 1.24153244, 0.57119018, 0.02916753], + [14.61464119, 12.96784878, 11.54541874, 9.24142551, 8.30717278, 7.49001646, 5.85520077, 4.45427561, 3.1956799, 2.19988537, 1.24153244, 0.57119018, 0.02916753], + [14.61464119, 12.96784878, 11.54541874, 9.24142551, 8.30717278, 7.49001646, 6.14220476, 4.86714602, 3.75677586, 2.84484982, 1.84880662, 1.08895338, 0.52423614, 0.02916753], + [14.61464119, 13.76078796, 12.2308979, 10.90732002, 9.24142551, 8.30717278, 7.49001646, 6.14220476, 4.86714602, 3.75677586, 2.84484982, 1.84880662, 1.08895338, 0.52423614, 0.02916753], + [14.61464119, 13.76078796, 12.2308979, 10.90732002, 9.24142551, 8.30717278, 7.49001646, 6.44769001, 5.58536053, 4.45427561, 3.32507086, 2.45070267, 1.61558151, 0.95350921, 0.45573691, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 10.90732002, 9.24142551, 8.30717278, 7.49001646, 6.44769001, 5.58536053, 4.45427561, 3.32507086, 2.45070267, 1.61558151, 0.95350921, 0.45573691, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 10.90732002, 9.24142551, 8.30717278, 7.49001646, 6.77309084, 5.85520077, 4.86714602, 3.91689563, 3.07277966, 2.27973175, 1.56271636, 0.95350921, 0.45573691, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 11.54541874, 10.31284904, 9.24142551, 8.30717278, 7.49001646, 6.77309084, 5.85520077, 4.86714602, 3.91689563, 3.07277966, 2.27973175, 1.56271636, 0.95350921, 0.45573691, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 11.54541874, 10.31284904, 9.24142551, 8.75849152, 8.30717278, 7.49001646, 6.77309084, 5.85520077, 4.86714602, 3.91689563, 3.07277966, 2.27973175, 1.56271636, 0.95350921, 0.45573691, 0.02916753], + [14.61464119, 13.76078796, 12.96784878, 12.2308979, 11.54541874, 10.31284904, 9.24142551, 8.75849152, 8.30717278, 7.49001646, 6.77309084, 5.85520077, 5.09240818, 4.45427561, 3.60512662, 2.95596409, 2.19988537, 1.51179266, 0.89115214, 0.43325692, 0.02916753], + ], + 0.95: [ + [14.61464119, 6.77309084, 0.02916753], + [14.61464119, 6.77309084, 1.56271636, 0.02916753], + [14.61464119, 7.49001646, 2.84484982, 0.89115214, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 2.36326075, 0.803307, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 2.95596409, 1.56271636, 0.64427125, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 4.86714602, 2.95596409, 1.56271636, 0.64427125, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 4.86714602, 3.07277966, 1.91321158, 1.08895338, 0.50118381, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 5.85520077, 4.45427561, 3.07277966, 1.91321158, 1.08895338, 0.50118381, 0.02916753], + [14.61464119, 12.2308979, 8.75849152, 7.49001646, 5.85520077, 4.45427561, 3.07277966, 1.91321158, 1.08895338, 0.50118381, 0.02916753], + [14.61464119, 12.2308979, 8.75849152, 7.49001646, 5.85520077, 4.45427561, 3.1956799, 2.19988537, 1.41535246, 0.803307, 0.38853383, 0.02916753], + [14.61464119, 12.2308979, 8.75849152, 7.49001646, 5.85520077, 4.65472794, 3.46139455, 2.6383388, 1.84880662, 1.24153244, 0.72133851, 0.34370604, 0.02916753], + [14.61464119, 12.96784878, 10.90732002, 8.75849152, 7.49001646, 5.85520077, 4.65472794, 3.46139455, 2.6383388, 1.84880662, 1.24153244, 0.72133851, 0.34370604, 0.02916753], + [14.61464119, 12.96784878, 10.90732002, 8.75849152, 7.49001646, 6.14220476, 4.86714602, 3.75677586, 2.95596409, 2.19988537, 1.56271636, 1.05362725, 0.64427125, 0.32104823, 0.02916753], + [14.61464119, 12.96784878, 10.90732002, 8.75849152, 7.49001646, 6.44769001, 5.58536053, 4.65472794, 3.60512662, 2.95596409, 2.19988537, 1.56271636, 1.05362725, 0.64427125, 0.32104823, 0.02916753], + [14.61464119, 12.96784878, 11.54541874, 9.24142551, 8.30717278, 7.49001646, 6.44769001, 5.58536053, 4.65472794, 3.60512662, 2.95596409, 2.19988537, 1.56271636, 1.05362725, 0.64427125, 0.32104823, 0.02916753], + [14.61464119, 12.96784878, 11.54541874, 9.24142551, 8.30717278, 7.49001646, 6.44769001, 5.58536053, 4.65472794, 3.75677586, 3.07277966, 2.45070267, 1.78698075, 1.24153244, 0.83188516, 0.50118381, 0.22545385, 0.02916753], + [14.61464119, 12.96784878, 11.54541874, 9.24142551, 8.30717278, 7.49001646, 6.77309084, 5.85520077, 5.09240818, 4.45427561, 3.60512662, 2.95596409, 2.36326075, 1.72759056, 1.24153244, 0.83188516, 0.50118381, 0.22545385, 0.02916753], + [14.61464119, 13.76078796, 12.2308979, 10.90732002, 9.24142551, 8.30717278, 7.49001646, 6.77309084, 5.85520077, 5.09240818, 4.45427561, 3.60512662, 2.95596409, 2.36326075, 1.72759056, 1.24153244, 0.83188516, 0.50118381, 0.22545385, 0.02916753], + [14.61464119, 13.76078796, 12.2308979, 10.90732002, 9.24142551, 8.30717278, 7.49001646, 6.77309084, 5.85520077, 5.09240818, 4.45427561, 3.75677586, 3.07277966, 2.45070267, 1.91321158, 1.46270394, 1.05362725, 0.72133851, 0.43325692, 0.19894916, 0.02916753], + ], + 1.00: [ + [14.61464119, 1.56271636, 0.02916753], + [14.61464119, 6.77309084, 0.95350921, 0.02916753], + [14.61464119, 6.77309084, 2.36326075, 0.803307, 0.02916753], + [14.61464119, 7.11996698, 3.07277966, 1.56271636, 0.59516323, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 2.84484982, 1.41535246, 0.57119018, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 2.84484982, 1.61558151, 0.86115354, 0.38853383, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 4.86714602, 2.84484982, 1.61558151, 0.86115354, 0.38853383, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 4.86714602, 3.07277966, 1.98035145, 1.24153244, 0.72133851, 0.34370604, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 5.85520077, 4.45427561, 3.07277966, 1.98035145, 1.24153244, 0.72133851, 0.34370604, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 5.85520077, 4.45427561, 3.1956799, 2.27973175, 1.51179266, 0.95350921, 0.54755926, 0.25053367, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 5.85520077, 4.45427561, 3.1956799, 2.36326075, 1.61558151, 1.08895338, 0.72133851, 0.41087446, 0.17026083, 0.02916753], + [14.61464119, 11.54541874, 8.75849152, 7.49001646, 5.85520077, 4.45427561, 3.1956799, 2.36326075, 1.61558151, 1.08895338, 0.72133851, 0.41087446, 0.17026083, 0.02916753], + [14.61464119, 11.54541874, 8.75849152, 7.49001646, 5.85520077, 4.65472794, 3.60512662, 2.84484982, 2.12350607, 1.56271636, 1.08895338, 0.72133851, 0.41087446, 0.17026083, 0.02916753], + [14.61464119, 11.54541874, 8.75849152, 7.49001646, 5.85520077, 4.65472794, 3.60512662, 2.84484982, 2.19988537, 1.61558151, 1.162866, 0.803307, 0.50118381, 0.27464288, 0.09824532, 0.02916753], + [14.61464119, 11.54541874, 8.75849152, 7.49001646, 5.85520077, 4.65472794, 3.75677586, 3.07277966, 2.45070267, 1.84880662, 1.36964464, 1.01931262, 0.72133851, 0.45573691, 0.25053367, 0.09824532, 0.02916753], + [14.61464119, 11.54541874, 8.75849152, 7.49001646, 6.14220476, 5.09240818, 4.26497746, 3.46139455, 2.84484982, 2.19988537, 1.67050016, 1.24153244, 0.92192322, 0.64427125, 0.43325692, 0.25053367, 0.09824532, 0.02916753], + [14.61464119, 11.54541874, 8.75849152, 7.49001646, 6.14220476, 5.09240818, 4.26497746, 3.60512662, 2.95596409, 2.45070267, 1.91321158, 1.51179266, 1.12534678, 0.83188516, 0.59516323, 0.38853383, 0.22545385, 0.09824532, 0.02916753], + [14.61464119, 12.2308979, 9.24142551, 8.30717278, 7.49001646, 6.14220476, 5.09240818, 4.26497746, 3.60512662, 2.95596409, 2.45070267, 1.91321158, 1.51179266, 1.12534678, 0.83188516, 0.59516323, 0.38853383, 0.22545385, 0.09824532, 0.02916753], + [14.61464119, 12.2308979, 9.24142551, 8.30717278, 7.49001646, 6.77309084, 5.85520077, 5.09240818, 4.26497746, 3.60512662, 2.95596409, 2.45070267, 1.91321158, 1.51179266, 1.12534678, 0.83188516, 0.59516323, 0.38853383, 0.22545385, 0.09824532, 0.02916753], + ], + 1.05: [ + [14.61464119, 0.95350921, 0.02916753], + [14.61464119, 6.77309084, 0.89115214, 0.02916753], + [14.61464119, 6.77309084, 2.05039096, 0.72133851, 0.02916753], + [14.61464119, 6.77309084, 2.84484982, 1.28281462, 0.52423614, 0.02916753], + [14.61464119, 6.77309084, 3.07277966, 1.61558151, 0.803307, 0.34370604, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 2.84484982, 1.56271636, 0.803307, 0.34370604, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 2.84484982, 1.61558151, 0.95350921, 0.52423614, 0.22545385, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 3.07277966, 1.98035145, 1.24153244, 0.74807048, 0.41087446, 0.17026083, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 3.1956799, 2.27973175, 1.51179266, 0.95350921, 0.59516323, 0.34370604, 0.13792117, 0.02916753], + [14.61464119, 7.49001646, 5.09240818, 3.46139455, 2.45070267, 1.61558151, 1.08895338, 0.72133851, 0.45573691, 0.25053367, 0.09824532, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 5.09240818, 3.46139455, 2.45070267, 1.61558151, 1.08895338, 0.72133851, 0.45573691, 0.25053367, 0.09824532, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 5.85520077, 4.45427561, 3.1956799, 2.36326075, 1.61558151, 1.08895338, 0.72133851, 0.45573691, 0.25053367, 0.09824532, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 5.85520077, 4.45427561, 3.1956799, 2.45070267, 1.72759056, 1.24153244, 0.86115354, 0.59516323, 0.38853383, 0.22545385, 0.09824532, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 5.85520077, 4.65472794, 3.60512662, 2.84484982, 2.19988537, 1.61558151, 1.162866, 0.83188516, 0.59516323, 0.38853383, 0.22545385, 0.09824532, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 5.85520077, 4.65472794, 3.60512662, 2.84484982, 2.19988537, 1.67050016, 1.28281462, 0.95350921, 0.72133851, 0.52423614, 0.34370604, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 5.85520077, 4.65472794, 3.60512662, 2.95596409, 2.36326075, 1.84880662, 1.41535246, 1.08895338, 0.83188516, 0.61951244, 0.45573691, 0.32104823, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 5.85520077, 4.65472794, 3.60512662, 2.95596409, 2.45070267, 1.91321158, 1.51179266, 1.20157266, 0.95350921, 0.74807048, 0.57119018, 0.43325692, 0.29807833, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 11.54541874, 8.30717278, 7.11996698, 5.85520077, 4.65472794, 3.60512662, 2.95596409, 2.45070267, 1.91321158, 1.51179266, 1.20157266, 0.95350921, 0.74807048, 0.57119018, 0.43325692, 0.29807833, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 11.54541874, 8.30717278, 7.11996698, 5.85520077, 4.65472794, 3.60512662, 2.95596409, 2.45070267, 1.98035145, 1.61558151, 1.32549286, 1.08895338, 0.86115354, 0.69515091, 0.54755926, 0.41087446, 0.29807833, 0.19894916, 0.09824532, 0.02916753], + ], + 1.10: [ + [14.61464119, 0.89115214, 0.02916753], + [14.61464119, 2.36326075, 0.72133851, 0.02916753], + [14.61464119, 5.85520077, 1.61558151, 0.57119018, 0.02916753], + [14.61464119, 6.77309084, 2.45070267, 1.08895338, 0.45573691, 0.02916753], + [14.61464119, 6.77309084, 2.95596409, 1.56271636, 0.803307, 0.34370604, 0.02916753], + [14.61464119, 6.77309084, 3.07277966, 1.61558151, 0.89115214, 0.4783645, 0.19894916, 0.02916753], + [14.61464119, 6.77309084, 3.07277966, 1.84880662, 1.08895338, 0.64427125, 0.34370604, 0.13792117, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 2.84484982, 1.61558151, 0.95350921, 0.54755926, 0.27464288, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 2.95596409, 1.91321158, 1.24153244, 0.803307, 0.4783645, 0.25053367, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 3.07277966, 2.05039096, 1.41535246, 0.95350921, 0.64427125, 0.41087446, 0.22545385, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 3.1956799, 2.27973175, 1.61558151, 1.12534678, 0.803307, 0.54755926, 0.36617002, 0.22545385, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 3.32507086, 2.45070267, 1.72759056, 1.24153244, 0.89115214, 0.64427125, 0.45573691, 0.32104823, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 5.09240818, 3.60512662, 2.84484982, 2.05039096, 1.51179266, 1.08895338, 0.803307, 0.59516323, 0.43325692, 0.29807833, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 5.09240818, 3.60512662, 2.84484982, 2.12350607, 1.61558151, 1.24153244, 0.95350921, 0.72133851, 0.54755926, 0.41087446, 0.29807833, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 5.85520077, 4.45427561, 3.1956799, 2.45070267, 1.84880662, 1.41535246, 1.08895338, 0.83188516, 0.64427125, 0.50118381, 0.36617002, 0.25053367, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 5.85520077, 4.45427561, 3.1956799, 2.45070267, 1.91321158, 1.51179266, 1.20157266, 0.95350921, 0.74807048, 0.59516323, 0.45573691, 0.34370604, 0.25053367, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 5.85520077, 4.45427561, 3.46139455, 2.84484982, 2.19988537, 1.72759056, 1.36964464, 1.08895338, 0.86115354, 0.69515091, 0.54755926, 0.43325692, 0.34370604, 0.25053367, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 5.85520077, 4.45427561, 3.46139455, 2.84484982, 2.19988537, 1.72759056, 1.36964464, 1.08895338, 0.86115354, 0.69515091, 0.54755926, 0.43325692, 0.34370604, 0.25053367, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 11.54541874, 7.49001646, 5.85520077, 4.45427561, 3.46139455, 2.84484982, 2.19988537, 1.72759056, 1.36964464, 1.08895338, 0.89115214, 0.72133851, 0.59516323, 0.4783645, 0.38853383, 0.29807833, 0.22545385, 0.17026083, 0.09824532, 0.02916753], + ], + 1.15: [ + [14.61464119, 0.83188516, 0.02916753], + [14.61464119, 1.84880662, 0.59516323, 0.02916753], + [14.61464119, 5.85520077, 1.56271636, 0.52423614, 0.02916753], + [14.61464119, 5.85520077, 1.91321158, 0.83188516, 0.34370604, 0.02916753], + [14.61464119, 5.85520077, 2.45070267, 1.24153244, 0.59516323, 0.25053367, 0.02916753], + [14.61464119, 5.85520077, 2.84484982, 1.51179266, 0.803307, 0.41087446, 0.17026083, 0.02916753], + [14.61464119, 5.85520077, 2.84484982, 1.56271636, 0.89115214, 0.50118381, 0.25053367, 0.09824532, 0.02916753], + [14.61464119, 6.77309084, 3.07277966, 1.84880662, 1.12534678, 0.72133851, 0.43325692, 0.22545385, 0.09824532, 0.02916753], + [14.61464119, 6.77309084, 3.07277966, 1.91321158, 1.24153244, 0.803307, 0.52423614, 0.34370604, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 2.95596409, 1.91321158, 1.24153244, 0.803307, 0.52423614, 0.34370604, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 3.07277966, 2.05039096, 1.36964464, 0.95350921, 0.69515091, 0.4783645, 0.32104823, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 3.07277966, 2.12350607, 1.51179266, 1.08895338, 0.803307, 0.59516323, 0.43325692, 0.29807833, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 3.07277966, 2.12350607, 1.51179266, 1.08895338, 0.803307, 0.59516323, 0.45573691, 0.34370604, 0.25053367, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 3.07277966, 2.19988537, 1.61558151, 1.24153244, 0.95350921, 0.74807048, 0.59516323, 0.45573691, 0.34370604, 0.25053367, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 3.1956799, 2.45070267, 1.78698075, 1.32549286, 1.01931262, 0.803307, 0.64427125, 0.50118381, 0.38853383, 0.29807833, 0.22545385, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 3.1956799, 2.45070267, 1.78698075, 1.32549286, 1.01931262, 0.803307, 0.64427125, 0.52423614, 0.41087446, 0.32104823, 0.25053367, 0.19894916, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 3.1956799, 2.45070267, 1.84880662, 1.41535246, 1.12534678, 0.89115214, 0.72133851, 0.59516323, 0.4783645, 0.38853383, 0.32104823, 0.25053367, 0.19894916, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 3.1956799, 2.45070267, 1.84880662, 1.41535246, 1.12534678, 0.89115214, 0.72133851, 0.59516323, 0.50118381, 0.41087446, 0.34370604, 0.27464288, 0.22545385, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.86714602, 3.1956799, 2.45070267, 1.84880662, 1.41535246, 1.12534678, 0.89115214, 0.72133851, 0.59516323, 0.50118381, 0.41087446, 0.34370604, 0.29807833, 0.25053367, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + ], + 1.20: [ + [14.61464119, 0.803307, 0.02916753], + [14.61464119, 1.56271636, 0.52423614, 0.02916753], + [14.61464119, 2.36326075, 0.92192322, 0.36617002, 0.02916753], + [14.61464119, 2.84484982, 1.24153244, 0.59516323, 0.25053367, 0.02916753], + [14.61464119, 5.85520077, 2.05039096, 0.95350921, 0.45573691, 0.17026083, 0.02916753], + [14.61464119, 5.85520077, 2.45070267, 1.24153244, 0.64427125, 0.29807833, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.45070267, 1.36964464, 0.803307, 0.45573691, 0.25053367, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.84484982, 1.61558151, 0.95350921, 0.59516323, 0.36617002, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.84484982, 1.67050016, 1.08895338, 0.74807048, 0.50118381, 0.32104823, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.95596409, 1.84880662, 1.24153244, 0.83188516, 0.59516323, 0.41087446, 0.27464288, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 3.07277966, 1.98035145, 1.36964464, 0.95350921, 0.69515091, 0.50118381, 0.36617002, 0.25053367, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 6.77309084, 3.46139455, 2.36326075, 1.56271636, 1.08895338, 0.803307, 0.59516323, 0.45573691, 0.34370604, 0.25053367, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 6.77309084, 3.46139455, 2.45070267, 1.61558151, 1.162866, 0.86115354, 0.64427125, 0.50118381, 0.38853383, 0.29807833, 0.22545385, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.65472794, 3.07277966, 2.12350607, 1.51179266, 1.08895338, 0.83188516, 0.64427125, 0.50118381, 0.38853383, 0.29807833, 0.22545385, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.65472794, 3.07277966, 2.12350607, 1.51179266, 1.08895338, 0.83188516, 0.64427125, 0.50118381, 0.41087446, 0.32104823, 0.25053367, 0.19894916, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.65472794, 3.07277966, 2.12350607, 1.51179266, 1.08895338, 0.83188516, 0.64427125, 0.50118381, 0.41087446, 0.34370604, 0.27464288, 0.22545385, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.65472794, 3.07277966, 2.19988537, 1.61558151, 1.20157266, 0.92192322, 0.72133851, 0.57119018, 0.45573691, 0.36617002, 0.29807833, 0.25053367, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.65472794, 3.07277966, 2.19988537, 1.61558151, 1.24153244, 0.95350921, 0.74807048, 0.59516323, 0.4783645, 0.38853383, 0.32104823, 0.27464288, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 7.49001646, 4.65472794, 3.07277966, 2.19988537, 1.61558151, 1.24153244, 0.95350921, 0.74807048, 0.59516323, 0.50118381, 0.41087446, 0.34370604, 0.29807833, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + ], + 1.25: [ + [14.61464119, 0.72133851, 0.02916753], + [14.61464119, 1.56271636, 0.50118381, 0.02916753], + [14.61464119, 2.05039096, 0.803307, 0.32104823, 0.02916753], + [14.61464119, 2.36326075, 0.95350921, 0.43325692, 0.17026083, 0.02916753], + [14.61464119, 2.84484982, 1.24153244, 0.59516323, 0.27464288, 0.09824532, 0.02916753], + [14.61464119, 3.07277966, 1.51179266, 0.803307, 0.43325692, 0.22545385, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.36326075, 1.24153244, 0.72133851, 0.41087446, 0.22545385, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.45070267, 1.36964464, 0.83188516, 0.52423614, 0.34370604, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.84484982, 1.61558151, 0.98595673, 0.64427125, 0.43325692, 0.27464288, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.84484982, 1.67050016, 1.08895338, 0.74807048, 0.52423614, 0.36617002, 0.25053367, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.84484982, 1.72759056, 1.162866, 0.803307, 0.59516323, 0.45573691, 0.34370604, 0.25053367, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.95596409, 1.84880662, 1.24153244, 0.86115354, 0.64427125, 0.4783645, 0.36617002, 0.27464288, 0.19894916, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.95596409, 1.84880662, 1.28281462, 0.92192322, 0.69515091, 0.52423614, 0.41087446, 0.32104823, 0.25053367, 0.19894916, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.95596409, 1.91321158, 1.32549286, 0.95350921, 0.72133851, 0.54755926, 0.43325692, 0.34370604, 0.27464288, 0.22545385, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.95596409, 1.91321158, 1.32549286, 0.95350921, 0.72133851, 0.57119018, 0.45573691, 0.36617002, 0.29807833, 0.25053367, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.95596409, 1.91321158, 1.32549286, 0.95350921, 0.74807048, 0.59516323, 0.4783645, 0.38853383, 0.32104823, 0.27464288, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 3.07277966, 2.05039096, 1.41535246, 1.05362725, 0.803307, 0.61951244, 0.50118381, 0.41087446, 0.34370604, 0.29807833, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 3.07277966, 2.05039096, 1.41535246, 1.05362725, 0.803307, 0.64427125, 0.52423614, 0.43325692, 0.36617002, 0.32104823, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 3.07277966, 2.05039096, 1.46270394, 1.08895338, 0.83188516, 0.66947293, 0.54755926, 0.45573691, 0.38853383, 0.34370604, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + ], + 1.30: [ + [14.61464119, 0.72133851, 0.02916753], + [14.61464119, 1.24153244, 0.43325692, 0.02916753], + [14.61464119, 1.56271636, 0.59516323, 0.22545385, 0.02916753], + [14.61464119, 1.84880662, 0.803307, 0.36617002, 0.13792117, 0.02916753], + [14.61464119, 2.36326075, 1.01931262, 0.52423614, 0.25053367, 0.09824532, 0.02916753], + [14.61464119, 2.84484982, 1.36964464, 0.74807048, 0.41087446, 0.22545385, 0.09824532, 0.02916753], + [14.61464119, 3.07277966, 1.56271636, 0.89115214, 0.54755926, 0.34370604, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 3.07277966, 1.61558151, 0.95350921, 0.61951244, 0.41087446, 0.27464288, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.45070267, 1.36964464, 0.83188516, 0.54755926, 0.36617002, 0.25053367, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.45070267, 1.41535246, 0.92192322, 0.64427125, 0.45573691, 0.34370604, 0.25053367, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.6383388, 1.56271636, 1.01931262, 0.72133851, 0.50118381, 0.36617002, 0.27464288, 0.19894916, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.84484982, 1.61558151, 1.05362725, 0.74807048, 0.54755926, 0.41087446, 0.32104823, 0.25053367, 0.19894916, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.84484982, 1.61558151, 1.08895338, 0.77538133, 0.57119018, 0.43325692, 0.34370604, 0.27464288, 0.22545385, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.84484982, 1.61558151, 1.08895338, 0.803307, 0.59516323, 0.45573691, 0.36617002, 0.29807833, 0.25053367, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.84484982, 1.61558151, 1.08895338, 0.803307, 0.59516323, 0.4783645, 0.38853383, 0.32104823, 0.27464288, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.84484982, 1.72759056, 1.162866, 0.83188516, 0.64427125, 0.50118381, 0.41087446, 0.34370604, 0.29807833, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.84484982, 1.72759056, 1.162866, 0.83188516, 0.64427125, 0.52423614, 0.43325692, 0.36617002, 0.32104823, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.84484982, 1.78698075, 1.24153244, 0.92192322, 0.72133851, 0.57119018, 0.45573691, 0.38853383, 0.34370604, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.84484982, 1.78698075, 1.24153244, 0.92192322, 0.72133851, 0.57119018, 0.4783645, 0.41087446, 0.36617002, 0.32104823, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + ], + 1.35: [ + [14.61464119, 0.69515091, 0.02916753], + [14.61464119, 0.95350921, 0.34370604, 0.02916753], + [14.61464119, 1.56271636, 0.57119018, 0.19894916, 0.02916753], + [14.61464119, 1.61558151, 0.69515091, 0.29807833, 0.09824532, 0.02916753], + [14.61464119, 1.84880662, 0.83188516, 0.43325692, 0.22545385, 0.09824532, 0.02916753], + [14.61464119, 2.45070267, 1.162866, 0.64427125, 0.36617002, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 2.84484982, 1.36964464, 0.803307, 0.50118381, 0.32104823, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 2.84484982, 1.41535246, 0.83188516, 0.54755926, 0.36617002, 0.25053367, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 2.84484982, 1.56271636, 0.95350921, 0.64427125, 0.45573691, 0.32104823, 0.22545385, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 2.84484982, 1.56271636, 0.95350921, 0.64427125, 0.45573691, 0.34370604, 0.25053367, 0.19894916, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 3.07277966, 1.61558151, 1.01931262, 0.72133851, 0.52423614, 0.38853383, 0.29807833, 0.22545385, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 3.07277966, 1.61558151, 1.01931262, 0.72133851, 0.52423614, 0.41087446, 0.32104823, 0.25053367, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 3.07277966, 1.61558151, 1.05362725, 0.74807048, 0.54755926, 0.43325692, 0.34370604, 0.27464288, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 3.07277966, 1.72759056, 1.12534678, 0.803307, 0.59516323, 0.45573691, 0.36617002, 0.29807833, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 3.07277966, 1.72759056, 1.12534678, 0.803307, 0.59516323, 0.4783645, 0.38853383, 0.32104823, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.45070267, 1.51179266, 1.01931262, 0.74807048, 0.57119018, 0.45573691, 0.36617002, 0.32104823, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.6383388, 1.61558151, 1.08895338, 0.803307, 0.61951244, 0.50118381, 0.41087446, 0.34370604, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.6383388, 1.61558151, 1.08895338, 0.803307, 0.64427125, 0.52423614, 0.43325692, 0.36617002, 0.32104823, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 5.85520077, 2.6383388, 1.61558151, 1.08895338, 0.803307, 0.64427125, 0.52423614, 0.45573691, 0.38853383, 0.34370604, 0.32104823, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + ], + 1.40: [ + [14.61464119, 0.59516323, 0.02916753], + [14.61464119, 0.95350921, 0.34370604, 0.02916753], + [14.61464119, 1.08895338, 0.43325692, 0.13792117, 0.02916753], + [14.61464119, 1.56271636, 0.64427125, 0.27464288, 0.09824532, 0.02916753], + [14.61464119, 1.61558151, 0.803307, 0.43325692, 0.22545385, 0.09824532, 0.02916753], + [14.61464119, 2.05039096, 0.95350921, 0.54755926, 0.34370604, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 2.45070267, 1.24153244, 0.72133851, 0.43325692, 0.27464288, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 2.45070267, 1.24153244, 0.74807048, 0.50118381, 0.34370604, 0.25053367, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 2.45070267, 1.28281462, 0.803307, 0.52423614, 0.36617002, 0.27464288, 0.19894916, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.45070267, 1.28281462, 0.803307, 0.54755926, 0.38853383, 0.29807833, 0.22545385, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.84484982, 1.41535246, 0.86115354, 0.59516323, 0.43325692, 0.32104823, 0.25053367, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.84484982, 1.51179266, 0.95350921, 0.64427125, 0.45573691, 0.34370604, 0.27464288, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.84484982, 1.51179266, 0.95350921, 0.64427125, 0.4783645, 0.36617002, 0.29807833, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.84484982, 1.56271636, 0.98595673, 0.69515091, 0.52423614, 0.41087446, 0.34370604, 0.29807833, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.84484982, 1.56271636, 1.01931262, 0.72133851, 0.54755926, 0.43325692, 0.36617002, 0.32104823, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.84484982, 1.61558151, 1.05362725, 0.74807048, 0.57119018, 0.45573691, 0.38853383, 0.34370604, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.84484982, 1.61558151, 1.08895338, 0.803307, 0.61951244, 0.50118381, 0.41087446, 0.36617002, 0.32104823, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.84484982, 1.61558151, 1.08895338, 0.803307, 0.61951244, 0.50118381, 0.43325692, 0.38853383, 0.34370604, 0.32104823, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.84484982, 1.61558151, 1.08895338, 0.803307, 0.64427125, 0.52423614, 0.45573691, 0.41087446, 0.36617002, 0.34370604, 0.32104823, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + ], + 1.45: [ + [14.61464119, 0.59516323, 0.02916753], + [14.61464119, 0.803307, 0.25053367, 0.02916753], + [14.61464119, 0.95350921, 0.34370604, 0.09824532, 0.02916753], + [14.61464119, 1.24153244, 0.54755926, 0.25053367, 0.09824532, 0.02916753], + [14.61464119, 1.56271636, 0.72133851, 0.36617002, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 1.61558151, 0.803307, 0.45573691, 0.27464288, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 1.91321158, 0.95350921, 0.57119018, 0.36617002, 0.25053367, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 2.19988537, 1.08895338, 0.64427125, 0.41087446, 0.27464288, 0.19894916, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.45070267, 1.24153244, 0.74807048, 0.50118381, 0.34370604, 0.25053367, 0.19894916, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.45070267, 1.24153244, 0.74807048, 0.50118381, 0.36617002, 0.27464288, 0.22545385, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.45070267, 1.28281462, 0.803307, 0.54755926, 0.41087446, 0.32104823, 0.25053367, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.45070267, 1.28281462, 0.803307, 0.57119018, 0.43325692, 0.34370604, 0.27464288, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.45070267, 1.28281462, 0.83188516, 0.59516323, 0.45573691, 0.36617002, 0.29807833, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.45070267, 1.28281462, 0.83188516, 0.59516323, 0.45573691, 0.36617002, 0.32104823, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.84484982, 1.51179266, 0.95350921, 0.69515091, 0.52423614, 0.41087446, 0.34370604, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.84484982, 1.51179266, 0.95350921, 0.69515091, 0.52423614, 0.43325692, 0.36617002, 0.32104823, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.84484982, 1.56271636, 0.98595673, 0.72133851, 0.54755926, 0.45573691, 0.38853383, 0.34370604, 0.32104823, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.84484982, 1.56271636, 1.01931262, 0.74807048, 0.57119018, 0.4783645, 0.41087446, 0.36617002, 0.34370604, 0.32104823, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.84484982, 1.56271636, 1.01931262, 0.74807048, 0.59516323, 0.50118381, 0.43325692, 0.38853383, 0.36617002, 0.34370604, 0.32104823, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + ], + 1.50: [ + [14.61464119, 0.54755926, 0.02916753], + [14.61464119, 0.803307, 0.25053367, 0.02916753], + [14.61464119, 0.86115354, 0.32104823, 0.09824532, 0.02916753], + [14.61464119, 1.24153244, 0.54755926, 0.25053367, 0.09824532, 0.02916753], + [14.61464119, 1.56271636, 0.72133851, 0.36617002, 0.19894916, 0.09824532, 0.02916753], + [14.61464119, 1.61558151, 0.803307, 0.45573691, 0.27464288, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 1.61558151, 0.83188516, 0.52423614, 0.34370604, 0.25053367, 0.17026083, 0.09824532, 0.02916753], + [14.61464119, 1.84880662, 0.95350921, 0.59516323, 0.38853383, 0.27464288, 0.19894916, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 1.84880662, 0.95350921, 0.59516323, 0.41087446, 0.29807833, 0.22545385, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 1.84880662, 0.95350921, 0.61951244, 0.43325692, 0.32104823, 0.25053367, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.19988537, 1.12534678, 0.72133851, 0.50118381, 0.36617002, 0.27464288, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.19988537, 1.12534678, 0.72133851, 0.50118381, 0.36617002, 0.29807833, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.36326075, 1.24153244, 0.803307, 0.57119018, 0.43325692, 0.34370604, 0.29807833, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.36326075, 1.24153244, 0.803307, 0.57119018, 0.43325692, 0.34370604, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.36326075, 1.24153244, 0.803307, 0.59516323, 0.45573691, 0.36617002, 0.32104823, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.36326075, 1.24153244, 0.803307, 0.59516323, 0.45573691, 0.38853383, 0.34370604, 0.32104823, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.45070267, 1.32549286, 0.86115354, 0.64427125, 0.50118381, 0.41087446, 0.36617002, 0.34370604, 0.32104823, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.45070267, 1.36964464, 0.92192322, 0.69515091, 0.54755926, 0.45573691, 0.41087446, 0.36617002, 0.34370604, 0.32104823, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + [14.61464119, 2.45070267, 1.41535246, 0.95350921, 0.72133851, 0.57119018, 0.4783645, 0.43325692, 0.38853383, 0.36617002, 0.34370604, 0.32104823, 0.29807833, 0.27464288, 0.25053367, 0.22545385, 0.19894916, 0.17026083, 0.13792117, 0.09824532, 0.02916753], + ], +} + +class GITSScheduler(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="GITSScheduler", + category="model/sampling/schedulers", + inputs=[ + io.Float.Input("coeff", default=1.20, min=0.80, max=1.50, step=0.05, advanced=True), + io.Int.Input("steps", default=10, min=2, max=1000), + io.Float.Input("denoise", default=1.0, min=0.0, max=1.0, step=0.01), + ], + outputs=[ + io.Sigmas.Output(), + ], + ) + + @classmethod + def execute(cls, coeff, steps, denoise): + total_steps = steps + if denoise < 1.0: + if denoise <= 0.0: + return io.NodeOutput(torch.FloatTensor([])) + total_steps = round(steps * denoise) + + if steps <= 20: + sigmas = NOISE_LEVELS[round(coeff, 2)][steps-2][:] + else: + sigmas = NOISE_LEVELS[round(coeff, 2)][-1][:] + sigmas = loglinear_interp(sigmas, steps + 1) + + sigmas = sigmas[-(total_steps + 1):] + sigmas[-1] = 0 + return io.NodeOutput(torch.FloatTensor(sigmas)) + + +class GITSSchedulerExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + GITSScheduler, + ] + + +async def comfy_entrypoint() -> GITSSchedulerExtension: + return GITSSchedulerExtension() diff --git a/ComfyUI/comfy_extras/nodes_glsl.py b/ComfyUI/comfy_extras/nodes_glsl.py new file mode 100644 index 0000000000000000000000000000000000000000..c7161973a64be376f454b9a6e53eddabd9449181 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_glsl.py @@ -0,0 +1,846 @@ +import os +import sys +import re +import ctypes +import logging +from typing import TypedDict + +import numpy as np +import torch + +import nodes +import comfy_angle +from comfy_api.latest import ComfyExtension, io, ui +from typing_extensions import override + +logger = logging.getLogger(__name__) + + +def _preload_angle(): + egl_path = comfy_angle.get_egl_path() + gles_path = comfy_angle.get_glesv2_path() + + if sys.platform == "win32": + angle_dir = comfy_angle.get_lib_dir() + os.add_dll_directory(angle_dir) + os.environ["PATH"] = angle_dir + os.pathsep + os.environ.get("PATH", "") + + mode = 0 if sys.platform == "win32" else ctypes.RTLD_GLOBAL + ctypes.CDLL(str(egl_path), mode=mode) + ctypes.CDLL(str(gles_path), mode=mode) + + +# Pre-load ANGLE *before* any PyOpenGL import so that the EGL platform +# plugin picks up ANGLE's libEGL / libGLESv2 instead of system libs. +_preload_angle() +os.environ.setdefault("PYOPENGL_PLATFORM", "egl") + + +import OpenGL +OpenGL.USE_ACCELERATE = False + + +def _patch_find_library(): + """PyOpenGL's EGL platform looks for 'EGL' and 'GLESv2' by short name + via ctypes.util.find_library, but ANGLE ships as 'libEGL' and + 'libGLESv2'. Patch find_library to return the full ANGLE paths so + PyOpenGL loads the same libraries we pre-loaded.""" + if sys.platform == "linux": + return + import ctypes.util + _orig = ctypes.util.find_library + def _patched(name): + if name == 'EGL': + return comfy_angle.get_egl_path() + if name == 'GLESv2': + return comfy_angle.get_glesv2_path() + return _orig(name) + ctypes.util.find_library = _patched + + +_patch_find_library() + +from OpenGL import EGL +from OpenGL import GLES3 as gl + +class SizeModeInput(TypedDict): + size_mode: str + width: int + height: int + + +MAX_IMAGES = 5 # u_image0-4 +MAX_UNIFORMS = 20 # u_float0-19, u_int0-19 +MAX_BOOLS = 10 # u_bool0-9 +MAX_CURVES = 4 # u_curve0-3 (1D LUT textures) +MAX_OUTPUTS = 4 # fragColor0-3 (MRT) + +# Vertex shader using gl_VertexID trick - no VBO needed. +# Draws a single triangle that covers the entire screen: +# +# (-1,3) +# /| +# / | <- visible area is the unit square from (-1,-1) to (1,1) +# / | parts outside get clipped away +# (-1,-1)---(3,-1) +# +# v_texCoord is computed from clip space: * 0.5 + 0.5 maps (-1,1) -> (0,1) +VERTEX_SHADER = """#version 300 es +out vec2 v_texCoord; +void main() { + vec2 verts[3] = vec2[](vec2(-1, -1), vec2(3, -1), vec2(-1, 3)); + v_texCoord = verts[gl_VertexID] * 0.5 + 0.5; + gl_Position = vec4(verts[gl_VertexID], 0, 1); +} +""" + +DEFAULT_FRAGMENT_SHADER = """#version 300 es +precision highp float; + +uniform sampler2D u_image0; +uniform vec2 u_resolution; + +in vec2 v_texCoord; +layout(location = 0) out vec4 fragColor0; + +void main() { + fragColor0 = texture(u_image0, v_texCoord); +} +""" + + + +def _egl_attribs(*values): + """Build an EGL_NONE-terminated EGLint attribute array.""" + vals = list(values) + [EGL.EGL_NONE] + return (ctypes.c_int32 * len(vals))(*vals) + + +# EGL platform extension constants +EGL_PLATFORM_ANGLE_ANGLE = 0x3202 +EGL_PLATFORM_ANGLE_TYPE_ANGLE = 0x3203 +EGL_PLATFORM_ANGLE_TYPE_VULKAN_ANGLE = 0x3450 +EGL_MESA_PLATFORM_SURFACELESS = 0x31DD + + +_eglGetPlatformDisplayEXT = None + +def _get_egl_platform_display_ext(platform, native_display, attribs): + """Call eglGetPlatformDisplayEXT via ctypes (extension, not in PyOpenGL).""" + global _eglGetPlatformDisplayEXT + if _eglGetPlatformDisplayEXT is None: + from OpenGL import platform as _plat + egl_lib = _plat.PLATFORM.EGL + _get_proc = egl_lib.eglGetProcAddress + _get_proc.restype = ctypes.c_void_p + _get_proc.argtypes = [ctypes.c_char_p] + ptr = _get_proc(b"eglGetPlatformDisplayEXT") + if not ptr: + return None + func_type = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_uint32, ctypes.c_void_p, ctypes.c_void_p) + _eglGetPlatformDisplayEXT = func_type(ptr) + + raw = _eglGetPlatformDisplayEXT(platform, native_display, attribs) + if not raw: + return None + return ctypes.cast(raw, EGL.EGLDisplay) + + +def _get_egl_display(): + """Get an EGL display, trying the default first then ANGLE's Vulkan + platform for headless environments without a display server.""" + failures = [] + + # Try the default display first (works when X11/Wayland is available) + display = EGL.eglGetDisplay(EGL.EGL_DEFAULT_DISPLAY) + if display: + major, minor = ctypes.c_int32(0), ctypes.c_int32(0) + try: + if EGL.eglInitialize(display, ctypes.byref(major), ctypes.byref(minor)): + return display, major.value, minor.value + except Exception as e: + failures.append(f"default: {e}") + + logger.info("Default EGL display unavailable, trying headless fallbacks") + + # Headless fallback strategies, tried in order: + headless_strategies = [ + ("surfaceless", EGL_MESA_PLATFORM_SURFACELESS, None, None), + ("ANGLE Vulkan", EGL_PLATFORM_ANGLE_ANGLE, None, + _egl_attribs(EGL_PLATFORM_ANGLE_TYPE_ANGLE, EGL_PLATFORM_ANGLE_TYPE_VULKAN_ANGLE)), + ] + + for name, platform, native_display, attribs in headless_strategies: + display = _get_egl_platform_display_ext(platform, native_display, attribs) + if not display: + failures.append(f"{name}: eglGetPlatformDisplayEXT returned no display") + continue + major, minor = ctypes.c_int32(0), ctypes.c_int32(0) + try: + if EGL.eglInitialize(display, ctypes.byref(major), ctypes.byref(minor)): + logger.info(f"Using EGL {name} platform (headless)") + return display, major.value, minor.value + failures.append(f"{name}: eglInitialize returned false") + except Exception as e: + failures.append(f"{name}: {e}") + continue + + details = "\n".join(f" - {f}" for f in failures) + raise RuntimeError( + "Failed to initialize EGL display.\n" + "No display server and no headless EGL platform available.\n" + f"Tried:\n{details}\n" + "Ensure GPU drivers are installed or set DISPLAY for a virtual framebuffer." + ) + + +def _gl_str(name): + """Get an OpenGL string parameter.""" + v = gl.glGetString(name) + if not v: + return "Unknown" + if isinstance(v, bytes): + return v.decode(errors="replace") + return ctypes.string_at(v).decode(errors="replace") + + +def _detect_output_count(source: str) -> int: + """Detect how many fragColor outputs are used in the shader. + + Returns the count of outputs needed (1 to MAX_OUTPUTS). + """ + matches = re.findall(r"fragColor(\d+)", source) + if not matches: + return 1 # Default to 1 output if none found + max_index = max(int(m) for m in matches) + return min(max_index + 1, MAX_OUTPUTS) + + +def _detect_pass_count(source: str) -> int: + """Detect multi-pass rendering from #pragma passes N directive. + + Returns the number of passes (1 if not specified). + """ + match = re.search(r'#pragma\s+passes\s+(\d+)', source) + if match: + return max(1, int(match.group(1))) + return 1 + + +class GLContext: + """Manages an OpenGL ES 3.0 context via EGL/ANGLE (singleton).""" + + _instance = None + _initialized = False + + def __new__(cls): + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self): + if GLContext._initialized: + return + + import time + start = time.perf_counter() + + self._display = None + self._surface = None + self._context = None + self._vao = None + + try: + self._display, self._egl_major, self._egl_minor = _get_egl_display() + + if not EGL.eglBindAPI(EGL.EGL_OPENGL_ES_API): + raise RuntimeError("eglBindAPI(EGL_OPENGL_ES_API) failed") + + config = EGL.EGLConfig() + n_configs = ctypes.c_int32(0) + if not EGL.eglChooseConfig( + self._display, + _egl_attribs( + EGL.EGL_RENDERABLE_TYPE, EGL.EGL_OPENGL_ES3_BIT, + EGL.EGL_SURFACE_TYPE, EGL.EGL_PBUFFER_BIT, + EGL.EGL_RED_SIZE, 8, EGL.EGL_GREEN_SIZE, 8, + EGL.EGL_BLUE_SIZE, 8, EGL.EGL_ALPHA_SIZE, 8, + ), + ctypes.byref(config), 1, ctypes.byref(n_configs), + ) or n_configs.value == 0: + raise RuntimeError("eglChooseConfig() failed") + + self._surface = EGL.eglCreatePbufferSurface( + self._display, config, + _egl_attribs(EGL.EGL_WIDTH, 64, EGL.EGL_HEIGHT, 64), + ) + if not self._surface: + raise RuntimeError("eglCreatePbufferSurface() failed") + + self._context = EGL.eglCreateContext( + self._display, config, EGL.EGL_NO_CONTEXT, + _egl_attribs(EGL.EGL_CONTEXT_CLIENT_VERSION, 3), + ) + if not self._context: + raise RuntimeError("eglCreateContext() failed") + + if not EGL.eglMakeCurrent(self._display, self._surface, self._surface, self._context): + raise RuntimeError("eglMakeCurrent() failed") + + self._vao = gl.glGenVertexArrays(1) + gl.glBindVertexArray(self._vao) + + except Exception: + self._cleanup() + raise + + elapsed = (time.perf_counter() - start) * 1000 + + renderer = _gl_str(gl.GL_RENDERER) + vendor = _gl_str(gl.GL_VENDOR) + version = _gl_str(gl.GL_VERSION) + + GLContext._initialized = True + logger.info(f"GLSL context initialized in {elapsed:.1f}ms - EGL {self._egl_major}.{self._egl_minor}, {renderer} ({vendor}), GL {version}") + + def make_current(self): + if not EGL.eglMakeCurrent(self._display, self._surface, self._surface, self._context): + err = EGL.eglGetError() + raise RuntimeError(f"eglMakeCurrent() failed (EGL error: 0x{err:04X})") + if self._vao is not None: + gl.glBindVertexArray(self._vao) + + def _cleanup(self): + if not self._display: + return + try: + if self._vao is not None: + gl.glDeleteVertexArrays(1, [self._vao]) + self._vao = None + except Exception: + pass + try: + EGL.eglMakeCurrent(self._display, EGL.EGL_NO_SURFACE, EGL.EGL_NO_SURFACE, EGL.EGL_NO_CONTEXT) + except Exception: + pass + try: + if self._context: + EGL.eglDestroyContext(self._display, self._context) + except Exception: + pass + try: + if self._surface: + EGL.eglDestroySurface(self._display, self._surface) + except Exception: + pass + try: + EGL.eglTerminate(self._display) + except Exception: + pass + self._display = None + + +def _compile_shader(source: str, shader_type: int) -> int: + """Compile a shader and return its ID.""" + shader = gl.glCreateShader(shader_type) + gl.glShaderSource(shader, source) + gl.glCompileShader(shader) + + if not gl.glGetShaderiv(shader, gl.GL_COMPILE_STATUS): + error = gl.glGetShaderInfoLog(shader) + if isinstance(error, bytes): + error = error.decode(errors="replace") + gl.glDeleteShader(shader) + raise RuntimeError(f"Shader compilation failed:\n{error}") + + return shader + + +def _create_program(vertex_source: str, fragment_source: str) -> int: + """Create and link a shader program.""" + vertex_shader = _compile_shader(vertex_source, gl.GL_VERTEX_SHADER) + try: + fragment_shader = _compile_shader(fragment_source, gl.GL_FRAGMENT_SHADER) + except RuntimeError: + gl.glDeleteShader(vertex_shader) + raise + + program = gl.glCreateProgram() + gl.glAttachShader(program, vertex_shader) + gl.glAttachShader(program, fragment_shader) + gl.glLinkProgram(program) + + gl.glDeleteShader(vertex_shader) + gl.glDeleteShader(fragment_shader) + + if not gl.glGetProgramiv(program, gl.GL_LINK_STATUS): + error = gl.glGetProgramInfoLog(program) + if isinstance(error, bytes): + error = error.decode(errors="replace") + gl.glDeleteProgram(program) + raise RuntimeError(f"Program linking failed:\n{error}") + + return program + + +def _render_shader_batch( + fragment_code: str, + width: int, + height: int, + image_batches: list[list[np.ndarray]], + floats: list[float], + ints: list[int], + bools: list[bool] | None = None, + curves: list[np.ndarray] | None = None, +) -> list[list[np.ndarray]]: + """ + Render a fragment shader for multiple batches efficiently. + + Compiles shader once, reuses framebuffer/textures across batches. + Supports multi-pass rendering via #pragma passes N directive. + + Args: + fragment_code: User's fragment shader code + width: Output width + height: Output height + image_batches: List of batches, each batch is a list of input images (H, W, C) float32 [0,1] + floats: List of float uniforms + ints: List of int uniforms + bools: List of bool uniforms (passed as int 0/1 to GLSL bool uniforms) + curves: List of 1D LUT arrays (float32) of arbitrary size for u_curve0-N + + Returns: + List of batch outputs, each is a list of output images (H, W, 4) float32 [0,1] + """ + import time + start_time = time.perf_counter() + + if not image_batches: + return [] + + ctx = GLContext() + ctx.make_current() + + # Detect how many outputs the shader actually uses + num_outputs = _detect_output_count(fragment_code) + + # Detect multi-pass rendering + num_passes = _detect_pass_count(fragment_code) + + if bools is None: + bools = [] + if curves is None: + curves = [] + + # Track resources for cleanup + program = None + fbo = None + output_textures = [] + input_textures = [] + curve_textures = [] + ping_pong_textures = [] + ping_pong_fbos = [] + + num_inputs = len(image_batches[0]) + + try: + # Compile shaders (once for all batches) + try: + program = _create_program(VERTEX_SHADER, fragment_code) + except RuntimeError: + logger.error(f"Fragment shader:\n{fragment_code}") + raise + + gl.glUseProgram(program) + + # Create framebuffer with only the needed color attachments + fbo = gl.glGenFramebuffers(1) + gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, fbo) + + draw_buffers = [] + for i in range(num_outputs): + tex = gl.glGenTextures(1) + output_textures.append(tex) + gl.glBindTexture(gl.GL_TEXTURE_2D, tex) + gl.glTexImage2D(gl.GL_TEXTURE_2D, 0, gl.GL_RGBA32F, width, height, 0, gl.GL_RGBA, gl.GL_FLOAT, None) + gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MIN_FILTER, gl.GL_LINEAR) + gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MAG_FILTER, gl.GL_LINEAR) + gl.glFramebufferTexture2D(gl.GL_FRAMEBUFFER, gl.GL_COLOR_ATTACHMENT0 + i, gl.GL_TEXTURE_2D, tex, 0) + draw_buffers.append(gl.GL_COLOR_ATTACHMENT0 + i) + + gl.glDrawBuffers(num_outputs, draw_buffers) + + if gl.glCheckFramebufferStatus(gl.GL_FRAMEBUFFER) != gl.GL_FRAMEBUFFER_COMPLETE: + raise RuntimeError("Framebuffer is not complete") + + # Create ping-pong resources for multi-pass rendering + if num_passes > 1: + for _ in range(2): + pp_tex = gl.glGenTextures(1) + ping_pong_textures.append(pp_tex) + gl.glBindTexture(gl.GL_TEXTURE_2D, pp_tex) + gl.glTexImage2D(gl.GL_TEXTURE_2D, 0, gl.GL_RGBA32F, width, height, 0, gl.GL_RGBA, gl.GL_FLOAT, None) + gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MIN_FILTER, gl.GL_LINEAR) + gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MAG_FILTER, gl.GL_LINEAR) + gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_S, gl.GL_CLAMP_TO_EDGE) + gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_T, gl.GL_CLAMP_TO_EDGE) + + pp_fbo = gl.glGenFramebuffers(1) + ping_pong_fbos.append(pp_fbo) + gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, pp_fbo) + gl.glFramebufferTexture2D(gl.GL_FRAMEBUFFER, gl.GL_COLOR_ATTACHMENT0, gl.GL_TEXTURE_2D, pp_tex, 0) + gl.glDrawBuffers(1, [gl.GL_COLOR_ATTACHMENT0]) + + if gl.glCheckFramebufferStatus(gl.GL_FRAMEBUFFER) != gl.GL_FRAMEBUFFER_COMPLETE: + raise RuntimeError("Ping-pong framebuffer is not complete") + + # Create input textures (reused for all batches) + for i in range(num_inputs): + tex = gl.glGenTextures(1) + input_textures.append(tex) + gl.glActiveTexture(gl.GL_TEXTURE0 + i) + gl.glBindTexture(gl.GL_TEXTURE_2D, tex) + gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MIN_FILTER, gl.GL_LINEAR) + gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MAG_FILTER, gl.GL_LINEAR) + gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_S, gl.GL_CLAMP_TO_EDGE) + gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_T, gl.GL_CLAMP_TO_EDGE) + + loc = gl.glGetUniformLocation(program, f"u_image{i}") + if loc >= 0: + gl.glUniform1i(loc, i) + + # Set static uniforms (once for all batches) + loc = gl.glGetUniformLocation(program, "u_resolution") + if loc >= 0: + gl.glUniform2f(loc, float(width), float(height)) + + for i, v in enumerate(floats): + loc = gl.glGetUniformLocation(program, f"u_float{i}") + if loc >= 0: + gl.glUniform1f(loc, v) + + for i, v in enumerate(ints): + loc = gl.glGetUniformLocation(program, f"u_int{i}") + if loc >= 0: + gl.glUniform1i(loc, v) + + for i, v in enumerate(bools): + loc = gl.glGetUniformLocation(program, f"u_bool{i}") + if loc >= 0: + gl.glUniform1i(loc, 1 if v else 0) + + # Create 1D LUT textures for curves (bound after image texture units) + for i, lut in enumerate(curves): + tex = gl.glGenTextures(1) + curve_textures.append(tex) + unit = MAX_IMAGES + i + gl.glActiveTexture(gl.GL_TEXTURE0 + unit) + gl.glBindTexture(gl.GL_TEXTURE_2D, tex) + gl.glTexImage2D(gl.GL_TEXTURE_2D, 0, gl.GL_R32F, len(lut), 1, 0, gl.GL_RED, gl.GL_FLOAT, lut) + gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MIN_FILTER, gl.GL_LINEAR) + gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MAG_FILTER, gl.GL_LINEAR) + gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_S, gl.GL_CLAMP_TO_EDGE) + gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_T, gl.GL_CLAMP_TO_EDGE) + + loc = gl.glGetUniformLocation(program, f"u_curve{i}") + if loc >= 0: + gl.glUniform1i(loc, unit) + + # Get u_pass uniform location for multi-pass + pass_loc = gl.glGetUniformLocation(program, "u_pass") + + gl.glViewport(0, 0, width, height) + gl.glDisable(gl.GL_BLEND) # Ensure no alpha blending - write output directly + + # Process each batch + all_batch_outputs = [] + for images in image_batches: + # Update input textures with this batch's images + for i, img in enumerate(images): + gl.glActiveTexture(gl.GL_TEXTURE0 + i) + gl.glBindTexture(gl.GL_TEXTURE_2D, input_textures[i]) + + # Flip vertically for GL coordinates, ensure RGBA + h, w, c = img.shape + if c == 3: + img_upload = np.empty((h, w, 4), dtype=np.float32) + img_upload[:, :, :3] = img[::-1, :, :] + img_upload[:, :, 3] = 1.0 + else: + img_upload = np.ascontiguousarray(img[::-1, :, :]) + + gl.glTexImage2D(gl.GL_TEXTURE_2D, 0, gl.GL_RGBA32F, w, h, 0, gl.GL_RGBA, gl.GL_FLOAT, img_upload) + + if num_passes == 1: + # Single pass - render directly to output FBO + gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, fbo) + if pass_loc >= 0: + gl.glUniform1i(pass_loc, 0) + gl.glClearColor(0, 0, 0, 0) + gl.glClear(gl.GL_COLOR_BUFFER_BIT) + gl.glDrawArrays(gl.GL_TRIANGLES, 0, 3) + else: + # Multi-pass rendering with ping-pong + for p in range(num_passes): + is_last_pass = (p == num_passes - 1) + + # Set pass uniform + if pass_loc >= 0: + gl.glUniform1i(pass_loc, p) + + if is_last_pass: + # Last pass renders to the main output FBO + gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, fbo) + else: + # Intermediate passes render to ping-pong FBO + target_fbo = ping_pong_fbos[p % 2] + gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, target_fbo) + + # Set input texture for this pass + gl.glActiveTexture(gl.GL_TEXTURE0) + if p == 0: + # First pass reads from original input + gl.glBindTexture(gl.GL_TEXTURE_2D, input_textures[0]) + else: + # Subsequent passes read from previous pass output + source_tex = ping_pong_textures[(p - 1) % 2] + gl.glBindTexture(gl.GL_TEXTURE_2D, source_tex) + + gl.glClearColor(0, 0, 0, 0) + gl.glClear(gl.GL_COLOR_BUFFER_BIT) + gl.glDrawArrays(gl.GL_TRIANGLES, 0, 3) + + # Read back outputs for this batch + gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, fbo) + batch_outputs = [] + for i in range(num_outputs): + gl.glReadBuffer(gl.GL_COLOR_ATTACHMENT0 + i) + buf = np.empty((height, width, 4), dtype=np.float32) + gl.glReadPixels(0, 0, width, height, gl.GL_RGBA, gl.GL_FLOAT, buf) + batch_outputs.append(buf[::-1, :, :].copy()) + + # Pad with black images for unused outputs + black_img = np.zeros((height, width, 4), dtype=np.float32) + for _ in range(num_outputs, MAX_OUTPUTS): + batch_outputs.append(black_img) + + all_batch_outputs.append(batch_outputs) + + elapsed = (time.perf_counter() - start_time) * 1000 + num_batches = len(image_batches) + pass_info = f", {num_passes} passes" if num_passes > 1 else "" + logger.info(f"GLSL shader executed in {elapsed:.1f}ms ({num_batches} batch{'es' if num_batches != 1 else ''}, {width}x{height}{pass_info})") + + return all_batch_outputs + + finally: + # Unbind before deleting + gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, 0) + gl.glUseProgram(0) + + if input_textures: + gl.glDeleteTextures(len(input_textures), input_textures) + if curve_textures: + gl.glDeleteTextures(len(curve_textures), curve_textures) + if output_textures: + gl.glDeleteTextures(len(output_textures), output_textures) + if ping_pong_textures: + gl.glDeleteTextures(len(ping_pong_textures), ping_pong_textures) + if fbo is not None: + gl.glDeleteFramebuffers(1, [fbo]) + if ping_pong_fbos: + gl.glDeleteFramebuffers(len(ping_pong_fbos), ping_pong_fbos) + if program is not None: + gl.glDeleteProgram(program) + +class GLSLShader(io.ComfyNode): + + @classmethod + def define_schema(cls) -> io.Schema: + image_template = io.Autogrow.TemplatePrefix( + io.Image.Input("image"), + prefix="image", + min=1, + max=MAX_IMAGES, + ) + + float_template = io.Autogrow.TemplatePrefix( + io.Float.Input("float", default=0.0), + prefix="u_float", + min=0, + max=MAX_UNIFORMS, + ) + + int_template = io.Autogrow.TemplatePrefix( + io.Int.Input("int", default=0), + prefix="u_int", + min=0, + max=MAX_UNIFORMS, + ) + + bool_template = io.Autogrow.TemplatePrefix( + io.Boolean.Input("bool", default=False), + prefix="u_bool", + min=0, + max=MAX_BOOLS, + ) + + curve_template = io.Autogrow.TemplatePrefix( + io.Curve.Input("curve"), + prefix="u_curve", + min=0, + max=MAX_CURVES, + ) + + return io.Schema( + node_id="GLSLShader", + display_name="GLSL Shader", + category="image/shader", + description=( + "Apply GLSL ES fragment shaders to images. " + "u_resolution (vec2) is always available." + ), + is_experimental=True, + has_intermediate_output=True, + inputs=[ + io.String.Input( + "fragment_shader", + default=DEFAULT_FRAGMENT_SHADER, + multiline=True, + tooltip="GLSL fragment shader source code (GLSL ES 3.00 / WebGL 2.0 compatible)", + ), + io.DynamicCombo.Input( + "size_mode", + options=[ + io.DynamicCombo.Option("from_input", []), + io.DynamicCombo.Option( + "custom", + [ + io.Int.Input( + "width", + default=512, + min=1, + max=nodes.MAX_RESOLUTION, + ), + io.Int.Input( + "height", + default=512, + min=1, + max=nodes.MAX_RESOLUTION, + ), + ], + ), + ], + tooltip="Output size: 'from_input' uses first input image dimensions, 'custom' allows manual size", + ), + io.Autogrow.Input("images", template=image_template, tooltip=f"Images are available as u_image0-{MAX_IMAGES-1} (sampler2D) in the shader code"), + io.Autogrow.Input("floats", template=float_template, tooltip=f"Floats are available as u_float0-{MAX_UNIFORMS-1} in the shader code"), + io.Autogrow.Input("ints", template=int_template, tooltip=f"Ints are available as u_int0-{MAX_UNIFORMS-1} in the shader code"), + io.Autogrow.Input("bools", template=bool_template, tooltip=f"Booleans are available as u_bool0-{MAX_BOOLS-1} (bool) in the shader code"), + io.Autogrow.Input("curves", template=curve_template, tooltip=f"Curves are available as u_curve0-{MAX_CURVES-1} (sampler2D, 1D LUT) in the shader code. Sample with texture(u_curve0, vec2(x, 0.5)).r"), + ], + outputs=[ + io.Image.Output(display_name="IMAGE0", tooltip="Available via layout(location = 0) out vec4 fragColor0 in the shader code"), + io.Image.Output(display_name="IMAGE1", tooltip="Available via layout(location = 1) out vec4 fragColor1 in the shader code"), + io.Image.Output(display_name="IMAGE2", tooltip="Available via layout(location = 2) out vec4 fragColor2 in the shader code"), + io.Image.Output(display_name="IMAGE3", tooltip="Available via layout(location = 3) out vec4 fragColor3 in the shader code"), + ], + ) + + @classmethod + def execute( + cls, + fragment_shader: str, + size_mode: SizeModeInput, + images: io.Autogrow.Type, + floats: io.Autogrow.Type = None, + ints: io.Autogrow.Type = None, + bools: io.Autogrow.Type = None, + curves: io.Autogrow.Type = None, + **kwargs, + ) -> io.NodeOutput: + + image_list = [v for v in images.values() if v is not None] + float_list = ( + [v if v is not None else 0.0 for v in floats.values()] if floats else [] + ) + int_list = [v if v is not None else 0 for v in ints.values()] if ints else [] + bool_list = [v if v is not None else False for v in bools.values()] if bools else [] + + curve_luts = [v.to_lut().astype(np.float32) for v in curves.values() if v is not None] if curves else [] + + if not image_list: + raise ValueError("At least one input image is required") + + # Determine output dimensions + if size_mode["size_mode"] == "custom": + out_width = size_mode["width"] + out_height = size_mode["height"] + else: + out_height, out_width = image_list[0].shape[1:3] + + batch_size = image_list[0].shape[0] + + # Prepare batches + image_batches = [] + for batch_idx in range(batch_size): + batch_images = [img_tensor[batch_idx].cpu().numpy().astype(np.float32) for img_tensor in image_list] + image_batches.append(batch_images) + + all_batch_outputs = _render_shader_batch( + fragment_shader, + out_width, + out_height, + image_batches, + float_list, + int_list, + bool_list, + curve_luts, + ) + + # Collect outputs into tensors + all_outputs = [[] for _ in range(MAX_OUTPUTS)] + for batch_outputs in all_batch_outputs: + for i, out_img in enumerate(batch_outputs): + all_outputs[i].append(torch.from_numpy(out_img)) + + output_tensors = [torch.stack(all_outputs[i], dim=0) for i in range(MAX_OUTPUTS)] + return io.NodeOutput( + *output_tensors, + ui=cls._build_ui_output(image_list, output_tensors[0]), + ) + + @classmethod + def _build_ui_output( + cls, image_list: list[torch.Tensor], output_batch: torch.Tensor + ) -> dict[str, list]: + """Build UI output with input and output images for client-side shader execution.""" + input_images_ui = [] + for img in image_list: + input_images_ui.extend(ui.ImageSaveHelper.save_images( + img, + filename_prefix="GLSLShader_input", + folder_type=io.FolderType.temp, + cls=None, + compress_level=1, + )) + + output_images_ui = ui.ImageSaveHelper.save_images( + output_batch, + filename_prefix="GLSLShader_output", + folder_type=io.FolderType.temp, + cls=None, + compress_level=1, + ) + + return {"input_images": input_images_ui, "images": output_images_ui} + + +class GLSLExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [GLSLShader] + + +async def comfy_entrypoint() -> GLSLExtension: + return GLSLExtension() diff --git a/ComfyUI/comfy_extras/nodes_hidream.py b/ComfyUI/comfy_extras/nodes_hidream.py new file mode 100644 index 0000000000000000000000000000000000000000..65248561bd60fddb00230d5b53dd077236a9a884 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_hidream.py @@ -0,0 +1,76 @@ +from typing_extensions import override + +import folder_paths +import comfy.sd +import comfy.model_management +from comfy_api.latest import ComfyExtension, io + + +class QuadrupleCLIPLoader(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="QuadrupleCLIPLoader", + display_name="Load CLIP (Quadruple)", + category="model/loaders", + description="Recipes:\nhidream: long clip-l, long clip-g, t5xxl, llama_8b_3.1_instruct", + inputs=[ + io.Combo.Input("clip_name1", options=folder_paths.get_filename_list("text_encoders")), + io.Combo.Input("clip_name2", options=folder_paths.get_filename_list("text_encoders")), + io.Combo.Input("clip_name3", options=folder_paths.get_filename_list("text_encoders")), + io.Combo.Input("clip_name4", options=folder_paths.get_filename_list("text_encoders")), + ], + outputs=[ + io.Clip.Output(), + ] + ) + + @classmethod + def execute(cls, clip_name1, clip_name2, clip_name3, clip_name4): + clip_path1 = folder_paths.get_full_path_or_raise("text_encoders", clip_name1) + clip_path2 = folder_paths.get_full_path_or_raise("text_encoders", clip_name2) + clip_path3 = folder_paths.get_full_path_or_raise("text_encoders", clip_name3) + clip_path4 = folder_paths.get_full_path_or_raise("text_encoders", clip_name4) + clip = comfy.sd.load_clip(ckpt_paths=[clip_path1, clip_path2, clip_path3, clip_path4], embedding_directory=folder_paths.get_folder_paths("embeddings")) + return io.NodeOutput(clip) + +class CLIPTextEncodeHiDream(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="CLIPTextEncodeHiDream", + display_name="CLIP Text Encode (HiDream)", + search_aliases=["hidream prompt"], + category="model/conditioning/hidream", + inputs=[ + io.Clip.Input("clip"), + io.String.Input("clip_l", multiline=True, dynamic_prompts=True), + io.String.Input("clip_g", multiline=True, dynamic_prompts=True), + io.String.Input("t5xxl", multiline=True, dynamic_prompts=True), + io.String.Input("llama", multiline=True, dynamic_prompts=True), + ], + outputs=[ + io.Conditioning.Output(), + ] + ) + + @classmethod + def execute(cls, clip, clip_l, clip_g, t5xxl, llama): + tokens = clip.tokenize(clip_g) + tokens["l"] = clip.tokenize(clip_l)["l"] + tokens["t5xxl"] = clip.tokenize(t5xxl)["t5xxl"] + tokens["llama"] = clip.tokenize(llama)["llama"] + return io.NodeOutput(clip.encode_from_tokens_scheduled(tokens)) + + +class HiDreamExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + QuadrupleCLIPLoader, + CLIPTextEncodeHiDream, + ] + + +async def comfy_entrypoint() -> HiDreamExtension: + return HiDreamExtension() diff --git a/ComfyUI/comfy_extras/nodes_hidream_o1.py b/ComfyUI/comfy_extras/nodes_hidream_o1.py new file mode 100644 index 0000000000000000000000000000000000000000..85693fce6913c6a0a8a6fbe24e29f6605825b69d --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_hidream_o1.py @@ -0,0 +1,256 @@ +from typing_extensions import override + +import torch + +import comfy.model_management +import comfy.patcher_extension +import node_helpers +from comfy_api.latest import ComfyExtension, io + + +class EmptyHiDreamO1LatentImage(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="EmptyHiDreamO1LatentImage", + display_name="Empty HiDream-O1 Latent Image", + category="model/latent/hidream", + description=( + "Empty pixel-space latent for HiDream-O1-Image. The model was " + "trained at ~4 megapixels; lower resolutions go off-distribution " + "and quality regresses noticeably. Trained resolutions: " + "2048x2048, 2304x1728, 1728x2304, 2560x1440, 1440x2560, " + "2496x1664, 1664x2496, 3104x1312, 1312x3104, 2304x1792, 1792x2304." + ), + inputs=[ + io.Int.Input(id="width", default=2048, min=64, max=4096, step=32), + io.Int.Input(id="height", default=2048, min=64, max=4096, step=32), + io.Int.Input(id="batch_size", default=1, min=1, max=64), + ], + outputs=[io.Latent().Output()], + ) + + @classmethod + def execute(cls, *, width: int, height: int, batch_size: int = 1) -> io.NodeOutput: + latent = torch.zeros( + (batch_size, 3, height, width), + device=comfy.model_management.intermediate_device(), + ) + return io.NodeOutput({"samples": latent}) + + +class HiDreamO1ReferenceImages(io.ComfyNode): + """Attach reference images to both positive and negative conditioning.""" + + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="HiDreamO1ReferenceImages", + display_name="HiDream-O1 Reference Images", + category="model/conditioning/hidream", + description=( + "Attach 1-10 reference images to conditioning, one for edit instruction" + "or multiple for subject-driven personalization." + ), + inputs=[ + io.Conditioning.Input(id="positive"), + io.Conditioning.Input(id="negative"), + io.Autogrow.Input( + "images", + template=io.Autogrow.TemplateNames( + io.Image.Input("image"), + names=[f"image_{i}" for i in range(1, 11)], + min=1, + ), + tooltip=("Reference images. 1 image = instruction edit; 2-10 images = multi reference." + ), + ), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + ], + ) + + @classmethod + def execute(cls, *, positive, negative, images: io.Autogrow.Type) -> io.NodeOutput: + refs = [images[f"image_{i}"] for i in range(1, 11) if f"image_{i}" in images] + positive = node_helpers.conditioning_set_values(positive, {"reference_latents": refs}, append=True) + negative = node_helpers.conditioning_set_values(negative, {"reference_latents": refs}, append=True) + return io.NodeOutput(positive, negative) + + +class HiDreamO1PatchSeamSmoothing(io.ComfyNode): + PATCH_SIZE = 32 + EDGE_FEATHER = 4 + + # Shift presets per (pattern, N). 8-pass = 4-quadrant + 4 quarter-patch offsets. + SHIFTS_BY_PATTERN = { + ("single_shift", 2): [(0, 0), (16, 16)], + ("single_shift", 4): [(0, 0), (16, 0), (0, 16), (16, 16)], + ("single_shift", 8): [(0, 0), (16, 0), (0, 16), (16, 16), + (8, 8), (24, 8), (8, 24), (24, 24)], + ("symmetric", 2): [(-8, -8), (8, 8)], + ("symmetric", 4): [(-8, -8), (8, -8), (-8, 8), (8, 8)], + ("symmetric", 8): [(-12, -12), (4, -12), (-12, 4), (4, 4), + (-4, -4), (12, -4), (-4, 12), (12, 12)], + } + RAMP_LEVELS = { + "2": [2], + "4": [4], + "ramp_2_4": [2, 4], + "ramp_2_4_8": [2, 4, 8], + } + + @staticmethod + def _hann_tile(cy: int, cx: int, size: int = 32) -> torch.Tensor: + """size x size Hann tile peaking at (cy, cx) within a patch.""" + half = size // 2 + yy = torch.arange(size).view(size, 1) + xx = torch.arange(size).view(1, size) + dy = ((yy - cy + half) % size) - half + dx = ((xx - cx + half) % size) - half + return 0.25 * (1 + torch.cos(torch.pi * dy / half)) * (1 + torch.cos(torch.pi * dx / half)) + + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="HiDreamO1PatchSeamSmoothing", + display_name="HiDream-O1 Patch Seam Smoothing", + category="model/patch/hidream", + is_experimental=True, + description=( + "Average the model output across multiple shifted patch-grid " + "positions during the late portion of sampling. Cancels seams." + ), + inputs=[ + io.Model.Input(id="model"), + io.Float.Input(id="start_percent", default=0.8, min=0.0, max=1.0, step=0.01, + tooltip="Sampling progress (0=start, 1=end) at which the blend turns ON.", + ), + io.Float.Input(id="end_percent", default=1.0, min=0.0, max=1.0, step=0.01, + tooltip="Sampling progress at which the blend turns OFF.", + ), + io.Combo.Input( + id="pattern", + options=["single_shift", "symmetric"], + default="single_shift", + tooltip="Shift layout. single_shift: one pass at the natural patch grid + others offset. symmetric: all passes off-grid, shifts split around origin.", + ), + io.Combo.Input( + id="passes", + options=["2", "4", "ramp_2_4", "ramp_2_4_8"], + default="2", + tooltip="Number of passes per gated step. 2/4 = fixed. ramp_*: pass count increases as sampling approaches end (more smoothing where seams are most visible).", + ), + io.Combo.Input( + id="blend", + options=["average", "window", "median"], + default="average", + tooltip="average: equal-weight mean. window: Hann-windowed weighting favoring each pass away from its patch boundaries. median: per-pixel median, rejects wraparound-outlier passes.", + ), + io.Float.Input(id="strength", default=1.0, min=0.0, max=1.0, step=0.01, + tooltip="Interpolation between the natural-grid pred (0) and the averaged result (1).", + ), + ], + outputs=[io.Model.Output()], + ) + + @classmethod + def execute(cls, *, model, start_percent: float, end_percent: float, pattern: str, passes: str, blend: str, strength: float) -> io.NodeOutput: + if strength <= 0.0 or end_percent <= start_percent: + return io.NodeOutput(model) + + P = cls.PATCH_SIZE + half = P // 2 + shift_levels = [cls.SHIFTS_BY_PATTERN[(pattern, n)] for n in cls.RAMP_LEVELS[passes]] + + if blend == "window": + window_tile_levels = [ + torch.stack([cls._hann_tile((half - sy) % P, (half - sx) % P, P) for sy, sx in lst], dim=0) + for lst in shift_levels + ] + else: + window_tile_levels = [None] * len(shift_levels) + + m = model.clone() + model_sampling = m.get_model_object("model_sampling") + multiplier = float(model_sampling.multiplier) + start_t = float(model_sampling.percent_to_sigma(start_percent)) * multiplier + end_t = float(model_sampling.percent_to_sigma(end_percent)) * multiplier + + edge_ramp_cache: dict = {} + + def get_edge_ramp(H: int, W: int, device, dtype) -> torch.Tensor: + key = (H, W, device, dtype) + cached = edge_ramp_cache.get(key) + if cached is not None: + return cached + feather = cls.EDGE_FEATHER + ys = torch.minimum(torch.arange(H, device=device, dtype=torch.float32), + (H - 1) - torch.arange(H, device=device, dtype=torch.float32)) + xs = torch.minimum(torch.arange(W, device=device, dtype=torch.float32), + (W - 1) - torch.arange(W, device=device, dtype=torch.float32)) + y_mask = ((ys - P) / feather).clamp(0, 1) + x_mask = ((xs - P) / feather).clamp(0, 1) + ramp = (y_mask[:, None] * x_mask[None, :]).to(dtype) + edge_ramp_cache[key] = ramp + return ramp + + def smoothing_wrapper(executor, *args, **kwargs): + x = args[0] + t = float(args[1][0]) + pred = executor(*args, **kwargs) + if not (end_t <= t <= start_t): + return pred + # Pick shift-level by sigma phase across the gated range. + if len(shift_levels) == 1: + level_idx = 0 + else: + phase = (start_t - t) / max(start_t - end_t, 1e-8) + level_idx = min(int(phase * len(shift_levels)), len(shift_levels) - 1) + shifts = shift_levels[level_idx] + window_tiles = window_tile_levels[level_idx] + + preds = [] + for sy, sx in shifts: + if sy == 0 and sx == 0: + preds.append(pred) + continue + x_rolled = torch.roll(x, shifts=(sy, sx), dims=(-2, -1)) + pred_rolled = executor(x_rolled, *args[1:], **kwargs) + preds.append(torch.roll(pred_rolled, shifts=(-sy, -sx), dims=(-2, -1))) + stacked = torch.stack(preds, dim=0) # (N, B, C, H, W) + _, _, _, H, W = stacked.shape + if blend == "window": + N = stacked.shape[0] + tiles = window_tiles.to(device=stacked.device, dtype=stacked.dtype) + w = tiles.repeat(1, H // P, W // P)[:, :H, :W] + sum_w = w.sum(dim=0, keepdim=True) + w = torch.where(sum_w < 1e-3, torch.full_like(w, 1.0 / N), w / sum_w.clamp(min=1e-8)) + avg = (stacked * w[:, None, None, :, :]).sum(dim=0) + elif blend == "median": + avg = torch.median(stacked, dim=0).values + else: + avg = stacked.mean(dim=0) + + # Mask out the P-px wraparound contamination strip at each edge. + mask = get_edge_ramp(H, W, pred.device, pred.dtype) + return pred * (1.0 - mask * strength) + avg * (mask * strength) + + m.add_wrapper_with_key(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, "hidream_o1_patch_seam_smoothing", smoothing_wrapper) + return io.NodeOutput(m) + + +class HiDreamO1Extension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + EmptyHiDreamO1LatentImage, + HiDreamO1ReferenceImages, + HiDreamO1PatchSeamSmoothing, + ] + + +async def comfy_entrypoint() -> HiDreamO1Extension: + return HiDreamO1Extension() diff --git a/ComfyUI/comfy_extras/nodes_hooks.py b/ComfyUI/comfy_extras/nodes_hooks.py new file mode 100644 index 0000000000000000000000000000000000000000..056369e8683e87aad53d4be800d54022489baaa7 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_hooks.py @@ -0,0 +1,750 @@ +from __future__ import annotations +from typing import TYPE_CHECKING, Union +import logging +import torch +from collections.abc import Iterable + +if TYPE_CHECKING: + from comfy.sd import CLIP + +import comfy.hooks +import comfy.sd +import comfy.utils +import folder_paths + +########################################### +# Mask, Combine, and Hook Conditioning +#------------------------------------------ +class PairConditioningSetProperties: + NodeId = 'PairConditioningSetProperties' + NodeName = 'Cond Pair Set Props' + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "positive_NEW": ("CONDITIONING", ), + "negative_NEW": ("CONDITIONING", ), + "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}), + "set_cond_area": (["default", "mask bounds"],), + }, + "optional": { + "mask": ("MASK", ), + "hooks": ("HOOKS",), + "timesteps": ("TIMESTEPS_RANGE",), + } + } + + EXPERIMENTAL = True + RETURN_TYPES = ("CONDITIONING", "CONDITIONING") + RETURN_NAMES = ("positive", "negative") + CATEGORY = "advanced/hooks/cond pair" + FUNCTION = "set_properties" + + def set_properties(self, positive_NEW, negative_NEW, + strength: float, set_cond_area: str, + mask: torch.Tensor=None, hooks: comfy.hooks.HookGroup=None, timesteps: tuple=None): + final_positive, final_negative = comfy.hooks.set_conds_props(conds=[positive_NEW, negative_NEW], + strength=strength, set_cond_area=set_cond_area, + mask=mask, hooks=hooks, timesteps_range=timesteps) + return (final_positive, final_negative) + +class PairConditioningSetPropertiesAndCombine: + NodeId = 'PairConditioningSetPropertiesAndCombine' + NodeName = 'Cond Pair Set Props Combine' + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "positive": ("CONDITIONING", ), + "negative": ("CONDITIONING", ), + "positive_NEW": ("CONDITIONING", ), + "negative_NEW": ("CONDITIONING", ), + "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}), + "set_cond_area": (["default", "mask bounds"],), + }, + "optional": { + "mask": ("MASK", ), + "hooks": ("HOOKS",), + "timesteps": ("TIMESTEPS_RANGE",), + } + } + + EXPERIMENTAL = True + RETURN_TYPES = ("CONDITIONING", "CONDITIONING") + RETURN_NAMES = ("positive", "negative") + CATEGORY = "advanced/hooks/cond pair" + FUNCTION = "set_properties" + + def set_properties(self, positive, negative, positive_NEW, negative_NEW, + strength: float, set_cond_area: str, + mask: torch.Tensor=None, hooks: comfy.hooks.HookGroup=None, timesteps: tuple=None): + final_positive, final_negative = comfy.hooks.set_conds_props_and_combine(conds=[positive, negative], new_conds=[positive_NEW, negative_NEW], + strength=strength, set_cond_area=set_cond_area, + mask=mask, hooks=hooks, timesteps_range=timesteps) + return (final_positive, final_negative) + +class ConditioningSetProperties: + NodeId = 'ConditioningSetProperties' + NodeName = 'Cond Set Props' + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "cond_NEW": ("CONDITIONING", ), + "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}), + "set_cond_area": (["default", "mask bounds"],), + }, + "optional": { + "mask": ("MASK", ), + "hooks": ("HOOKS",), + "timesteps": ("TIMESTEPS_RANGE",), + } + } + + EXPERIMENTAL = True + RETURN_TYPES = ("CONDITIONING",) + CATEGORY = "advanced/hooks/cond single" + FUNCTION = "set_properties" + + def set_properties(self, cond_NEW, + strength: float, set_cond_area: str, + mask: torch.Tensor=None, hooks: comfy.hooks.HookGroup=None, timesteps: tuple=None): + (final_cond,) = comfy.hooks.set_conds_props(conds=[cond_NEW], + strength=strength, set_cond_area=set_cond_area, + mask=mask, hooks=hooks, timesteps_range=timesteps) + return (final_cond,) + +class ConditioningSetPropertiesAndCombine: + NodeId = 'ConditioningSetPropertiesAndCombine' + NodeName = 'Cond Set Props Combine' + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "cond": ("CONDITIONING", ), + "cond_NEW": ("CONDITIONING", ), + "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}), + "set_cond_area": (["default", "mask bounds"],), + }, + "optional": { + "mask": ("MASK", ), + "hooks": ("HOOKS",), + "timesteps": ("TIMESTEPS_RANGE",), + } + } + + EXPERIMENTAL = True + RETURN_TYPES = ("CONDITIONING",) + CATEGORY = "advanced/hooks/cond single" + FUNCTION = "set_properties" + + def set_properties(self, cond, cond_NEW, + strength: float, set_cond_area: str, + mask: torch.Tensor=None, hooks: comfy.hooks.HookGroup=None, timesteps: tuple=None): + (final_cond,) = comfy.hooks.set_conds_props_and_combine(conds=[cond], new_conds=[cond_NEW], + strength=strength, set_cond_area=set_cond_area, + mask=mask, hooks=hooks, timesteps_range=timesteps) + return (final_cond,) + +class PairConditioningCombine: + NodeId = 'PairConditioningCombine' + NodeName = 'Cond Pair Combine' + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "positive_A": ("CONDITIONING",), + "negative_A": ("CONDITIONING",), + "positive_B": ("CONDITIONING",), + "negative_B": ("CONDITIONING",), + }, + } + + EXPERIMENTAL = True + RETURN_TYPES = ("CONDITIONING", "CONDITIONING") + RETURN_NAMES = ("positive", "negative") + CATEGORY = "advanced/hooks/cond pair" + FUNCTION = "combine" + + def combine(self, positive_A, negative_A, positive_B, negative_B): + final_positive, final_negative = comfy.hooks.set_conds_props_and_combine(conds=[positive_A, negative_A], new_conds=[positive_B, negative_B],) + return (final_positive, final_negative,) + +class PairConditioningSetDefaultAndCombine: + NodeId = 'PairConditioningSetDefaultCombine' + NodeName = 'Cond Pair Set Default Combine' + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "positive": ("CONDITIONING",), + "negative": ("CONDITIONING",), + "positive_DEFAULT": ("CONDITIONING",), + "negative_DEFAULT": ("CONDITIONING",), + }, + "optional": { + "hooks": ("HOOKS",), + } + } + + EXPERIMENTAL = True + RETURN_TYPES = ("CONDITIONING", "CONDITIONING") + RETURN_NAMES = ("positive", "negative") + CATEGORY = "advanced/hooks/cond pair" + FUNCTION = "set_default_and_combine" + + def set_default_and_combine(self, positive, negative, positive_DEFAULT, negative_DEFAULT, + hooks: comfy.hooks.HookGroup=None): + final_positive, final_negative = comfy.hooks.set_default_conds_and_combine(conds=[positive, negative], new_conds=[positive_DEFAULT, negative_DEFAULT], + hooks=hooks) + return (final_positive, final_negative) + +class ConditioningSetDefaultAndCombine: + NodeId = 'ConditioningSetDefaultCombine' + NodeName = 'Cond Set Default Combine' + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "cond": ("CONDITIONING",), + "cond_DEFAULT": ("CONDITIONING",), + }, + "optional": { + "hooks": ("HOOKS",), + } + } + + EXPERIMENTAL = True + RETURN_TYPES = ("CONDITIONING",) + CATEGORY = "advanced/hooks/cond single" + FUNCTION = "set_default_and_combine" + + def set_default_and_combine(self, cond, cond_DEFAULT, + hooks: comfy.hooks.HookGroup=None): + (final_conditioning,) = comfy.hooks.set_default_conds_and_combine(conds=[cond], new_conds=[cond_DEFAULT], + hooks=hooks) + return (final_conditioning,) + +class SetClipHooks: + NodeId = 'SetClipHooks' + NodeName = 'Set CLIP Hooks' + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "clip": ("CLIP",), + "apply_to_conds": ("BOOLEAN", {"default": True, "advanced": True}), + "schedule_clip": ("BOOLEAN", {"default": False, "advanced": True}) + }, + "optional": { + "hooks": ("HOOKS",) + } + } + + EXPERIMENTAL = True + RETURN_TYPES = ("CLIP",) + CATEGORY = "advanced/hooks/clip" + FUNCTION = "apply_hooks" + + def apply_hooks(self, clip: CLIP, schedule_clip: bool, apply_to_conds: bool, hooks: comfy.hooks.HookGroup=None): + if hooks is not None: + clip = clip.clone(disable_dynamic=True) + if apply_to_conds: + clip.apply_hooks_to_conds = hooks + clip.patcher.forced_hooks = hooks.clone() + clip.use_clip_schedule = schedule_clip + if not clip.use_clip_schedule: + clip.patcher.forced_hooks.set_keyframes_on_hooks(None) + clip.patcher.register_all_hook_patches(hooks, comfy.hooks.create_target_dict(comfy.hooks.EnumWeightTarget.Clip)) + return (clip,) + +class ConditioningTimestepsRange: + SEARCH_ALIASES = ["prompt scheduling", "timestep segments", "conditioning phases"] + NodeId = 'ConditioningTimestepsRange' + NodeName = 'Timesteps Range' + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}), + "end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001}) + }, + } + + EXPERIMENTAL = True + RETURN_TYPES = ("TIMESTEPS_RANGE", "TIMESTEPS_RANGE", "TIMESTEPS_RANGE") + RETURN_NAMES = ("TIMESTEPS_RANGE", "BEFORE_RANGE", "AFTER_RANGE") + CATEGORY = "advanced/hooks" + FUNCTION = "create_range" + + def create_range(self, start_percent: float, end_percent: float): + return ((start_percent, end_percent), (0.0, start_percent), (end_percent, 1.0)) +#------------------------------------------ +########################################### + + +########################################### +# Create Hooks +#------------------------------------------ +class CreateHookLora: + NodeId = 'CreateHookLora' + NodeName = 'Create Hook LoRA' + def __init__(self): + self.loaded_lora = None + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "lora_name": (folder_paths.get_filename_list("loras"), ), + "strength_model": ("FLOAT", {"default": 1.0, "min": -20.0, "max": 20.0, "step": 0.01}), + "strength_clip": ("FLOAT", {"default": 1.0, "min": -20.0, "max": 20.0, "step": 0.01}), + }, + "optional": { + "prev_hooks": ("HOOKS",) + } + } + + EXPERIMENTAL = True + RETURN_TYPES = ("HOOKS",) + CATEGORY = "advanced/hooks/create" + FUNCTION = "create_hook" + + def create_hook(self, lora_name: str, strength_model: float, strength_clip: float, prev_hooks: comfy.hooks.HookGroup=None): + if prev_hooks is None: + prev_hooks = comfy.hooks.HookGroup() + prev_hooks.clone() + + if strength_model == 0 and strength_clip == 0: + return (prev_hooks,) + + lora_path = folder_paths.get_full_path("loras", lora_name) + lora = None + if self.loaded_lora is not None: + if self.loaded_lora[0] == lora_path: + lora = self.loaded_lora[1] + else: + temp = self.loaded_lora + self.loaded_lora = None + del temp + + if lora is None: + lora = comfy.utils.load_torch_file(lora_path, safe_load=True) + self.loaded_lora = (lora_path, lora) + + hooks = comfy.hooks.create_hook_lora(lora=lora, strength_model=strength_model, strength_clip=strength_clip) + return (prev_hooks.clone_and_combine(hooks),) + +class CreateHookLoraModelOnly(CreateHookLora): + NodeId = 'CreateHookLoraModelOnly' + NodeName = 'Create Hook LoRA (MO)' + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "lora_name": (folder_paths.get_filename_list("loras"), ), + "strength_model": ("FLOAT", {"default": 1.0, "min": -20.0, "max": 20.0, "step": 0.01}), + }, + "optional": { + "prev_hooks": ("HOOKS",) + } + } + + EXPERIMENTAL = True + RETURN_TYPES = ("HOOKS",) + CATEGORY = "advanced/hooks/create" + FUNCTION = "create_hook_model_only" + + def create_hook_model_only(self, lora_name: str, strength_model: float, prev_hooks: comfy.hooks.HookGroup=None): + return self.create_hook(lora_name=lora_name, strength_model=strength_model, strength_clip=0, prev_hooks=prev_hooks) + +class CreateHookModelAsLora: + NodeId = 'CreateHookModelAsLora' + NodeName = 'Create Hook Model as LoRA' + + def __init__(self): + # when not None, will be in following format: + # (ckpt_path: str, weights_model: dict, weights_clip: dict) + self.loaded_weights = None + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "ckpt_name": (folder_paths.get_filename_list("checkpoints"), ), + "strength_model": ("FLOAT", {"default": 1.0, "min": -20.0, "max": 20.0, "step": 0.01}), + "strength_clip": ("FLOAT", {"default": 1.0, "min": -20.0, "max": 20.0, "step": 0.01}), + }, + "optional": { + "prev_hooks": ("HOOKS",) + } + } + + EXPERIMENTAL = True + RETURN_TYPES = ("HOOKS",) + CATEGORY = "advanced/hooks/create" + FUNCTION = "create_hook" + + def create_hook(self, ckpt_name: str, strength_model: float, strength_clip: float, + prev_hooks: comfy.hooks.HookGroup=None): + if prev_hooks is None: + prev_hooks = comfy.hooks.HookGroup() + prev_hooks.clone() + + ckpt_path = folder_paths.get_full_path("checkpoints", ckpt_name) + weights_model = None + weights_clip = None + if self.loaded_weights is not None: + if self.loaded_weights[0] == ckpt_path: + weights_model = self.loaded_weights[1] + weights_clip = self.loaded_weights[2] + else: + temp = self.loaded_weights + self.loaded_weights = None + del temp + + if weights_model is None: + out = comfy.sd.load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, embedding_directory=folder_paths.get_folder_paths("embeddings")) + weights_model = comfy.hooks.get_patch_weights_from_model(out[0]) + weights_clip = comfy.hooks.get_patch_weights_from_model(out[1].patcher if out[1] else out[1]) + self.loaded_weights = (ckpt_path, weights_model, weights_clip) + + hooks = comfy.hooks.create_hook_model_as_lora(weights_model=weights_model, weights_clip=weights_clip, + strength_model=strength_model, strength_clip=strength_clip) + return (prev_hooks.clone_and_combine(hooks),) + +class CreateHookModelAsLoraModelOnly(CreateHookModelAsLora): + NodeId = 'CreateHookModelAsLoraModelOnly' + NodeName = 'Create Hook Model as LoRA (MO)' + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "ckpt_name": (folder_paths.get_filename_list("checkpoints"), ), + "strength_model": ("FLOAT", {"default": 1.0, "min": -20.0, "max": 20.0, "step": 0.01}), + }, + "optional": { + "prev_hooks": ("HOOKS",) + } + } + + EXPERIMENTAL = True + RETURN_TYPES = ("HOOKS",) + CATEGORY = "advanced/hooks/create" + FUNCTION = "create_hook_model_only" + + def create_hook_model_only(self, ckpt_name: str, strength_model: float, + prev_hooks: comfy.hooks.HookGroup=None): + return self.create_hook(ckpt_name=ckpt_name, strength_model=strength_model, strength_clip=0.0, prev_hooks=prev_hooks) +#------------------------------------------ +########################################### + + +########################################### +# Schedule Hooks +#------------------------------------------ +class SetHookKeyframes: + NodeId = 'SetHookKeyframes' + NodeName = 'Set Hook Keyframes' + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "hooks": ("HOOKS",), + }, + "optional": { + "hook_kf": ("HOOK_KEYFRAMES",), + } + } + + EXPERIMENTAL = True + RETURN_TYPES = ("HOOKS",) + CATEGORY = "advanced/hooks/scheduling" + FUNCTION = "set_hook_keyframes" + + def set_hook_keyframes(self, hooks: comfy.hooks.HookGroup, hook_kf: comfy.hooks.HookKeyframeGroup=None): + if hook_kf is not None: + hooks = hooks.clone() + hooks.set_keyframes_on_hooks(hook_kf=hook_kf) + return (hooks,) + +class CreateHookKeyframe: + SEARCH_ALIASES = ["hook scheduling", "strength animation", "timed hook"] + NodeId = 'CreateHookKeyframe' + NodeName = 'Create Hook Keyframe' + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "strength_mult": ("FLOAT", {"default": 1.0, "min": -20.0, "max": 20.0, "step": 0.01}), + "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}), + }, + "optional": { + "prev_hook_kf": ("HOOK_KEYFRAMES",), + } + } + + EXPERIMENTAL = True + RETURN_TYPES = ("HOOK_KEYFRAMES",) + RETURN_NAMES = ("HOOK_KF",) + CATEGORY = "advanced/hooks/scheduling" + FUNCTION = "create_hook_keyframe" + + def create_hook_keyframe(self, strength_mult: float, start_percent: float, prev_hook_kf: comfy.hooks.HookKeyframeGroup=None): + if prev_hook_kf is None: + prev_hook_kf = comfy.hooks.HookKeyframeGroup() + prev_hook_kf = prev_hook_kf.clone() + keyframe = comfy.hooks.HookKeyframe(strength=strength_mult, start_percent=start_percent) + prev_hook_kf.add(keyframe) + return (prev_hook_kf,) + +class CreateHookKeyframesInterpolated: + SEARCH_ALIASES = ["ease hook strength", "smooth hook transition", "interpolate keyframes"] + NodeId = 'CreateHookKeyframesInterpolated' + NodeName = 'Create Hook Keyframes Interp.' + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "strength_start": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.001}, ), + "strength_end": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.001}, ), + "interpolation": (comfy.hooks.InterpolationMethod._LIST, ), + "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}), + "end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001}), + "keyframes_count": ("INT", {"default": 5, "min": 2, "max": 100, "step": 1}), + "print_keyframes": ("BOOLEAN", {"default": False, "advanced": True}), + }, + "optional": { + "prev_hook_kf": ("HOOK_KEYFRAMES",), + }, + } + + EXPERIMENTAL = True + RETURN_TYPES = ("HOOK_KEYFRAMES",) + RETURN_NAMES = ("HOOK_KF",) + CATEGORY = "advanced/hooks/scheduling" + FUNCTION = "create_hook_keyframes" + + def create_hook_keyframes(self, strength_start: float, strength_end: float, interpolation: str, + start_percent: float, end_percent: float, keyframes_count: int, + print_keyframes=False, prev_hook_kf: comfy.hooks.HookKeyframeGroup=None): + if prev_hook_kf is None: + prev_hook_kf = comfy.hooks.HookKeyframeGroup() + prev_hook_kf = prev_hook_kf.clone() + percents = comfy.hooks.InterpolationMethod.get_weights(num_from=start_percent, num_to=end_percent, length=keyframes_count, + method=comfy.hooks.InterpolationMethod.LINEAR) + strengths = comfy.hooks.InterpolationMethod.get_weights(num_from=strength_start, num_to=strength_end, length=keyframes_count, method=interpolation) + + is_first = True + for percent, strength in zip(percents, strengths): + guarantee_steps = 0 + if is_first: + guarantee_steps = 1 + is_first = False + prev_hook_kf.add(comfy.hooks.HookKeyframe(strength=strength, start_percent=percent, guarantee_steps=guarantee_steps)) + if print_keyframes: + logging.info(f"Hook Keyframe - start_percent:{percent} = {strength}") + return (prev_hook_kf,) + +class CreateHookKeyframesFromFloats: + SEARCH_ALIASES = ["batch keyframes", "strength list to keyframes"] + NodeId = 'CreateHookKeyframesFromFloats' + NodeName = 'Create Hook Keyframes From Floats' + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "floats_strength": ("FLOATS", {"default": -1, "min": -1, "step": 0.001, "forceInput": True}), + "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}), + "end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001}), + "print_keyframes": ("BOOLEAN", {"default": False, "advanced": True}), + }, + "optional": { + "prev_hook_kf": ("HOOK_KEYFRAMES",), + } + } + + EXPERIMENTAL = True + RETURN_TYPES = ("HOOK_KEYFRAMES",) + RETURN_NAMES = ("HOOK_KF",) + CATEGORY = "advanced/hooks/scheduling" + FUNCTION = "create_hook_keyframes" + + def create_hook_keyframes(self, floats_strength: Union[float, list[float]], + start_percent: float, end_percent: float, + prev_hook_kf: comfy.hooks.HookKeyframeGroup=None, print_keyframes=False): + if prev_hook_kf is None: + prev_hook_kf = comfy.hooks.HookKeyframeGroup() + prev_hook_kf = prev_hook_kf.clone() + if type(floats_strength) in (float, int): + floats_strength = [float(floats_strength)] + elif isinstance(floats_strength, Iterable): + pass + else: + raise Exception(f"floats_strength must be either an iterable input or a float, but was{type(floats_strength).__repr__}.") + percents = comfy.hooks.InterpolationMethod.get_weights(num_from=start_percent, num_to=end_percent, length=len(floats_strength), + method=comfy.hooks.InterpolationMethod.LINEAR) + + is_first = True + for percent, strength in zip(percents, floats_strength): + guarantee_steps = 0 + if is_first: + guarantee_steps = 1 + is_first = False + prev_hook_kf.add(comfy.hooks.HookKeyframe(strength=strength, start_percent=percent, guarantee_steps=guarantee_steps)) + if print_keyframes: + logging.info(f"Hook Keyframe - start_percent:{percent} = {strength}") + return (prev_hook_kf,) +#------------------------------------------ +########################################### + + +class SetModelHooksOnCond: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "conditioning": ("CONDITIONING",), + "hooks": ("HOOKS",), + }, + } + + EXPERIMENTAL = True + RETURN_TYPES = ("CONDITIONING",) + CATEGORY = "advanced/hooks/manual" + FUNCTION = "attach_hook" + + def attach_hook(self, conditioning, hooks: comfy.hooks.HookGroup): + return (comfy.hooks.set_hooks_for_conditioning(conditioning, hooks),) + + +########################################### +# Combine Hooks +#------------------------------------------ +class CombineHooks: + SEARCH_ALIASES = ["merge hooks"] + NodeId = 'CombineHooks2' + NodeName = 'Combine Hooks [2]' + @classmethod + def INPUT_TYPES(s): + return { + "required": { + }, + "optional": { + "hooks_A": ("HOOKS",), + "hooks_B": ("HOOKS",), + } + } + + EXPERIMENTAL = True + RETURN_TYPES = ("HOOKS",) + CATEGORY = "advanced/hooks/combine" + FUNCTION = "combine_hooks" + + def combine_hooks(self, + hooks_A: comfy.hooks.HookGroup=None, + hooks_B: comfy.hooks.HookGroup=None): + candidates = [hooks_A, hooks_B] + return (comfy.hooks.HookGroup.combine_all_hooks(candidates),) + +class CombineHooksFour: + NodeId = 'CombineHooks4' + NodeName = 'Combine Hooks [4]' + @classmethod + def INPUT_TYPES(s): + return { + "required": { + }, + "optional": { + "hooks_A": ("HOOKS",), + "hooks_B": ("HOOKS",), + "hooks_C": ("HOOKS",), + "hooks_D": ("HOOKS",), + } + } + + EXPERIMENTAL = True + RETURN_TYPES = ("HOOKS",) + CATEGORY = "advanced/hooks/combine" + FUNCTION = "combine_hooks" + + def combine_hooks(self, + hooks_A: comfy.hooks.HookGroup=None, + hooks_B: comfy.hooks.HookGroup=None, + hooks_C: comfy.hooks.HookGroup=None, + hooks_D: comfy.hooks.HookGroup=None): + candidates = [hooks_A, hooks_B, hooks_C, hooks_D] + return (comfy.hooks.HookGroup.combine_all_hooks(candidates),) + +class CombineHooksEight: + NodeId = 'CombineHooks8' + NodeName = 'Combine Hooks [8]' + @classmethod + def INPUT_TYPES(s): + return { + "required": { + }, + "optional": { + "hooks_A": ("HOOKS",), + "hooks_B": ("HOOKS",), + "hooks_C": ("HOOKS",), + "hooks_D": ("HOOKS",), + "hooks_E": ("HOOKS",), + "hooks_F": ("HOOKS",), + "hooks_G": ("HOOKS",), + "hooks_H": ("HOOKS",), + } + } + + EXPERIMENTAL = True + RETURN_TYPES = ("HOOKS",) + CATEGORY = "advanced/hooks/combine" + FUNCTION = "combine_hooks" + + def combine_hooks(self, + hooks_A: comfy.hooks.HookGroup=None, + hooks_B: comfy.hooks.HookGroup=None, + hooks_C: comfy.hooks.HookGroup=None, + hooks_D: comfy.hooks.HookGroup=None, + hooks_E: comfy.hooks.HookGroup=None, + hooks_F: comfy.hooks.HookGroup=None, + hooks_G: comfy.hooks.HookGroup=None, + hooks_H: comfy.hooks.HookGroup=None): + candidates = [hooks_A, hooks_B, hooks_C, hooks_D, hooks_E, hooks_F, hooks_G, hooks_H] + return (comfy.hooks.HookGroup.combine_all_hooks(candidates),) +#------------------------------------------ +########################################### + +node_list = [ + # Create + CreateHookLora, + CreateHookLoraModelOnly, + CreateHookModelAsLora, + CreateHookModelAsLoraModelOnly, + # Scheduling + SetHookKeyframes, + CreateHookKeyframe, + CreateHookKeyframesInterpolated, + CreateHookKeyframesFromFloats, + # Combine + CombineHooks, + CombineHooksFour, + CombineHooksEight, + # Attach + ConditioningSetProperties, + ConditioningSetPropertiesAndCombine, + PairConditioningSetProperties, + PairConditioningSetPropertiesAndCombine, + ConditioningSetDefaultAndCombine, + PairConditioningSetDefaultAndCombine, + PairConditioningCombine, + SetClipHooks, + # Other + ConditioningTimestepsRange, +] +NODE_CLASS_MAPPINGS = {} +NODE_DISPLAY_NAME_MAPPINGS = {} + +for node in node_list: + NODE_CLASS_MAPPINGS[node.NodeId] = node + NODE_DISPLAY_NAME_MAPPINGS[node.NodeId] = node.NodeName diff --git a/ComfyUI/comfy_extras/nodes_hunyuan.py b/ComfyUI/comfy_extras/nodes_hunyuan.py new file mode 100644 index 0000000000000000000000000000000000000000..ce2997245840e6f038b3e0c26ceb900cb6d9910a --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_hunyuan.py @@ -0,0 +1,438 @@ +import nodes +import node_helpers +import torch +import comfy.model_management +import comfy.model_patcher +import comfy.ops +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io +from comfy.ldm.hunyuan_video.upsampler import HunyuanVideo15SRModel +from comfy.ldm.lightricks.latent_upsampler import LatentUpsampler +import folder_paths +import json + +class CLIPTextEncodeHunyuanDiT(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="CLIPTextEncodeHunyuanDiT", + display_name="CLIP Text Encode (Hunyuan Image)", + category="model/conditioning/hunyuan image", + inputs=[ + io.Clip.Input("clip"), + io.String.Input("bert", multiline=True, dynamic_prompts=True), + io.String.Input("mt5xl", multiline=True, dynamic_prompts=True), + ], + outputs=[ + io.Conditioning.Output(), + ], + ) + + @classmethod + def execute(cls, clip, bert, mt5xl) -> io.NodeOutput: + tokens = clip.tokenize(bert) + tokens["mt5xl"] = clip.tokenize(mt5xl)["mt5xl"] + + return io.NodeOutput(clip.encode_from_tokens_scheduled(tokens)) + + encode = execute # TODO: remove + + +class EmptyHunyuanLatentVideo(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="EmptyHunyuanLatentVideo", + display_name="Empty HunyuanVideo 1.0 Latent", + category="model/latent/hunyuan video", + inputs=[ + io.Int.Input("width", default=848, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("length", default=25, min=1, max=nodes.MAX_RESOLUTION, step=4), + io.Int.Input("batch_size", default=1, min=1, max=4096), + ], + outputs=[ + io.Latent.Output(), + ], + ) + + @classmethod + def execute(cls, width, height, length, batch_size=1) -> io.NodeOutput: + latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device()) + return io.NodeOutput({"samples": latent, "downscale_ratio_spacial": 8}) + + generate = execute # TODO: remove + + +class EmptyHunyuanVideo15Latent(EmptyHunyuanLatentVideo): + @classmethod + def define_schema(cls): + schema = super().define_schema() + schema.node_id = "EmptyHunyuanVideo15Latent" + schema.display_name = "Empty HunyuanVideo 1.5 Latent" + schema.category = "model/latent/hunyuan video" + return schema + + @classmethod + def execute(cls, width, height, length, batch_size=1) -> io.NodeOutput: + # Using scale factor of 16 instead of 8 + latent = torch.zeros([batch_size, 32, ((length - 1) // 4) + 1, height // 16, width // 16], device=comfy.model_management.intermediate_device()) + return io.NodeOutput({"samples": latent, "downscale_ratio_spacial": 16}) + + +class HunyuanVideo15ImageToVideo(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="HunyuanVideo15ImageToVideo", + category="model/conditioning/hunyuan video", + inputs=[ + io.Conditioning.Input("positive"), + io.Conditioning.Input("negative"), + io.Vae.Input("vae"), + io.Int.Input("width", default=848, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("length", default=33, min=1, max=nodes.MAX_RESOLUTION, step=4), + io.Int.Input("batch_size", default=1, min=1, max=4096), + io.Image.Input("start_image", optional=True), + io.ClipVisionOutput.Input("clip_vision_output", optional=True), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + io.Latent.Output(display_name="latent"), + ], + ) + + @classmethod + def execute(cls, positive, negative, vae, width, height, length, batch_size, start_image=None, clip_vision_output=None) -> io.NodeOutput: + latent = torch.zeros([batch_size, 32, ((length - 1) // 4) + 1, height // 16, width // 16], device=comfy.model_management.intermediate_device()) + + if start_image is not None: + start_image = comfy.utils.common_upscale(start_image[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1) + + encoded = vae.encode(start_image[:, :, :, :3]) + concat_latent_image = torch.zeros((latent.shape[0], 32, latent.shape[2], latent.shape[3], latent.shape[4]), device=comfy.model_management.intermediate_device()) + concat_latent_image[:, :, :encoded.shape[2], :, :] = encoded + + mask = torch.ones((1, 1, latent.shape[2], concat_latent_image.shape[-2], concat_latent_image.shape[-1]), device=start_image.device, dtype=start_image.dtype) + mask[:, :, :((start_image.shape[0] - 1) // 4) + 1] = 0.0 + + positive = node_helpers.conditioning_set_values(positive, {"concat_latent_image": concat_latent_image, "concat_mask": mask}) + negative = node_helpers.conditioning_set_values(negative, {"concat_latent_image": concat_latent_image, "concat_mask": mask}) + + if clip_vision_output is not None: + positive = node_helpers.conditioning_set_values(positive, {"clip_vision_output": clip_vision_output}) + negative = node_helpers.conditioning_set_values(negative, {"clip_vision_output": clip_vision_output}) + + out_latent = {} + out_latent["samples"] = latent + return io.NodeOutput(positive, negative, out_latent) + + +class HunyuanVideo15SuperResolution(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="HunyuanVideo15SuperResolution", + display_name="Hunyuan Video 1.5 Super Resolution", + category="model/conditioning/hunyuan video", + inputs=[ + io.Conditioning.Input("positive"), + io.Conditioning.Input("negative"), + io.Vae.Input("vae", optional=True), + io.Image.Input("start_image", optional=True), + io.ClipVisionOutput.Input("clip_vision_output", optional=True), + io.Latent.Input("latent"), + io.Float.Input("noise_augmentation", default=0.70, min=0.0, max=1.0, step=0.01, advanced=True), + + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + io.Latent.Output(display_name="latent"), + ], + ) + + @classmethod + def execute(cls, positive, negative, latent, noise_augmentation, vae=None, start_image=None, clip_vision_output=None) -> io.NodeOutput: + in_latent = latent["samples"] + in_channels = in_latent.shape[1] + cond_latent = torch.zeros([in_latent.shape[0], in_channels * 2 + 2, in_latent.shape[-3], in_latent.shape[-2], in_latent.shape[-1]], device=comfy.model_management.intermediate_device()) + cond_latent[:, in_channels + 1 : 2 * in_channels + 1] = in_latent + cond_latent[:, 2 * in_channels + 1] = 1 + if start_image is not None: + start_image = comfy.utils.common_upscale(start_image.movedim(-1, 1), in_latent.shape[-1] * 16, in_latent.shape[-2] * 16, "bilinear", "center").movedim(1, -1) + encoded = vae.encode(start_image[:, :, :, :3]) + cond_latent[:, :in_channels, :encoded.shape[2], :, :] = encoded + cond_latent[:, in_channels + 1, 0] = 1 + + positive = node_helpers.conditioning_set_values(positive, {"concat_latent_image": cond_latent, "noise_augmentation": noise_augmentation}) + negative = node_helpers.conditioning_set_values(negative, {"concat_latent_image": cond_latent, "noise_augmentation": noise_augmentation}) + if clip_vision_output is not None: + positive = node_helpers.conditioning_set_values(positive, {"clip_vision_output": clip_vision_output}) + negative = node_helpers.conditioning_set_values(negative, {"clip_vision_output": clip_vision_output}) + + return io.NodeOutput(positive, negative, latent) + + +class LatentUpscaleModelLoader(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LatentUpscaleModelLoader", + display_name="Load Latent Upscale Model", + category="model/loaders", + inputs=[ + io.Combo.Input("model_name", options=folder_paths.get_filename_list("latent_upscale_models")), + ], + outputs=[ + io.LatentUpscaleModel.Output(), + ], + ) + + @classmethod + def execute(cls, model_name) -> io.NodeOutput: + model_path = folder_paths.get_full_path_or_raise("latent_upscale_models", model_name) + sd, metadata = comfy.utils.load_torch_file(model_path, safe_load=True, return_metadata=True) + + if "blocks.0.block.0.conv.weight" in sd: + config = { + "in_channels": sd["in_conv.conv.weight"].shape[1], + "out_channels": sd["out_conv.conv.weight"].shape[0], + "hidden_channels": sd["in_conv.conv.weight"].shape[0], + "num_blocks": len([k for k in sd.keys() if k.startswith("blocks.") and k.endswith(".block.0.conv.weight")]), + "global_residual": False, + } + model_type = "720p" + model = HunyuanVideo15SRModel(model_type, config) + model.load_sd(sd) + elif "up.0.block.0.conv1.conv.weight" in sd: + sd = {key.replace("nin_shortcut", "nin_shortcut.conv", 1): value for key, value in sd.items()} + config = { + "z_channels": sd["conv_in.conv.weight"].shape[1], + "out_channels": sd["conv_out.conv.weight"].shape[0], + "block_out_channels": tuple(sd[f"up.{i}.block.0.conv1.conv.weight"].shape[0] for i in range(len([k for k in sd.keys() if k.startswith("up.") and k.endswith(".block.0.conv1.conv.weight")]))), + } + model_type = "1080p" + model = HunyuanVideo15SRModel(model_type, config) + model.load_sd(sd) + elif "post_upsample_res_blocks.0.conv2.bias" in sd: + config = json.loads(metadata["config"]) + model = LatentUpsampler.from_config(config, operations=comfy.ops.disable_weight_init).to(dtype=comfy.model_management.vae_dtype(allowed_dtypes=[torch.bfloat16, torch.float32])) + comfy.model_management.archive_model_dtypes(model) + model_patcher = comfy.model_patcher.CoreModelPatcher(model, load_device=comfy.model_management.get_torch_device(), offload_device=comfy.model_management.unet_offload_device()) + model.load_state_dict(sd, assign=model_patcher.is_dynamic()) + model = model_patcher + + return io.NodeOutput(model) + + +class HunyuanVideo15LatentUpscaleWithModel(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="HunyuanVideo15LatentUpscaleWithModel", + display_name="Hunyuan Video 15 Latent Upscale With Model", + category="model/latent/hunyhuan video", + inputs=[ + io.LatentUpscaleModel.Input("model"), + io.Latent.Input("samples"), + io.Combo.Input("upscale_method", options=["nearest-exact", "bilinear", "area", "bicubic", "bislerp"], default="bilinear"), + io.Int.Input("width", default=1280, min=0, max=16384, step=8), + io.Int.Input("height", default=720, min=0, max=16384, step=8), + io.Combo.Input("crop", options=["disabled", "center"]), + ], + outputs=[ + io.Latent.Output(), + ], + ) + + @classmethod + def execute(cls, model, samples, upscale_method, width, height, crop) -> io.NodeOutput: + if width == 0 and height == 0: + return io.NodeOutput(samples) + else: + if width == 0: + height = max(64, height) + width = max(64, round(samples["samples"].shape[-1] * height / samples["samples"].shape[-2])) + elif height == 0: + width = max(64, width) + height = max(64, round(samples["samples"].shape[-2] * width / samples["samples"].shape[-1])) + else: + width = max(64, width) + height = max(64, height) + s = comfy.utils.common_upscale(samples["samples"], width // 16, height // 16, upscale_method, crop) + s = model.resample_latent(s) + return io.NodeOutput({"samples": s.cpu().float()}) + + +PROMPT_TEMPLATE_ENCODE_VIDEO_I2V = ( + "<|start_header_id|>system<|end_header_id|>\n\n\nDescribe the video by detailing the following aspects according to the reference image: " + "1. The main content and theme of the video." + "2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects." + "3. Actions, events, behaviors temporal relationships, physical movement changes of the objects." + "4. background environment, light, style and atmosphere." + "5. camera angles, movements, and transitions used in the video:<|eot_id|>\n\n" + "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>" + "<|start_header_id|>assistant<|end_header_id|>\n\n" +) + +class TextEncodeHunyuanVideo_ImageToVideo(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="TextEncodeHunyuanVideo_ImageToVideo", + category="model/conditioning/hunyuan video", + inputs=[ + io.Clip.Input("clip"), + io.ClipVisionOutput.Input("clip_vision_output"), + io.String.Input("prompt", multiline=True, dynamic_prompts=True), + io.Int.Input( + "image_interleave", + default=2, + min=1, + max=512, + tooltip="How much the image influences things vs the text prompt. Higher number means more influence from the text prompt.", + advanced=True, + ), + ], + outputs=[ + io.Conditioning.Output(), + ], + ) + + @classmethod + def execute(cls, clip, clip_vision_output, prompt, image_interleave) -> io.NodeOutput: + tokens = clip.tokenize(prompt, llama_template=PROMPT_TEMPLATE_ENCODE_VIDEO_I2V, image_embeds=clip_vision_output.mm_projected, image_interleave=image_interleave) + return io.NodeOutput(clip.encode_from_tokens_scheduled(tokens)) + + encode = execute # TODO: remove + + +class HunyuanImageToVideo(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="HunyuanImageToVideo", + category="model/conditioning/hunyuan video", + inputs=[ + io.Conditioning.Input("positive"), + io.Vae.Input("vae"), + io.Int.Input("width", default=848, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("length", default=53, min=1, max=nodes.MAX_RESOLUTION, step=4), + io.Int.Input("batch_size", default=1, min=1, max=4096), + io.Combo.Input("guidance_type", options=["v1 (concat)", "v2 (replace)", "custom"], advanced=True), + io.Image.Input("start_image", optional=True), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Latent.Output(display_name="latent"), + ], + ) + + @classmethod + def execute(cls, positive, vae, width, height, length, batch_size, guidance_type, start_image=None) -> io.NodeOutput: + latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device()) + out_latent = {} + + if start_image is not None: + start_image = comfy.utils.common_upscale(start_image[:length, :, :, :3].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1) + + concat_latent_image = vae.encode(start_image) + mask = torch.ones((1, 1, latent.shape[2], concat_latent_image.shape[-2], concat_latent_image.shape[-1]), device=start_image.device, dtype=start_image.dtype) + mask[:, :, :((start_image.shape[0] - 1) // 4) + 1] = 0.0 + + if guidance_type == "v1 (concat)": + cond = {"concat_latent_image": concat_latent_image, "concat_mask": mask} + elif guidance_type == "v2 (replace)": + cond = {'guiding_frame_index': 0} + latent[:, :, :concat_latent_image.shape[2]] = concat_latent_image + out_latent["noise_mask"] = mask + elif guidance_type == "custom": + cond = {"ref_latent": concat_latent_image} + + positive = node_helpers.conditioning_set_values(positive, cond) + + out_latent["samples"] = latent + return io.NodeOutput(positive, out_latent) + + encode = execute # TODO: remove + + +class EmptyHunyuanImageLatent(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="EmptyHunyuanImageLatent", + category="model/latent/hunyuan image", + inputs=[ + io.Int.Input("width", default=2048, min=64, max=nodes.MAX_RESOLUTION, step=32), + io.Int.Input("height", default=2048, min=64, max=nodes.MAX_RESOLUTION, step=32), + io.Int.Input("batch_size", default=1, min=1, max=4096), + ], + outputs=[ + io.Latent.Output(), + ], + ) + + @classmethod + def execute(cls, width, height, batch_size=1) -> io.NodeOutput: + latent = torch.zeros([batch_size, 64, height // 32, width // 32], device=comfy.model_management.intermediate_device()) + return io.NodeOutput({"samples":latent}) + + generate = execute # TODO: remove + + +class HunyuanRefinerLatent(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="HunyuanRefinerLatent", + display_name="Hunyuan Latent Refiner", + category="model/conditioning/hunyuan video", + inputs=[ + io.Conditioning.Input("positive"), + io.Conditioning.Input("negative"), + io.Latent.Input("latent"), + io.Float.Input("noise_augmentation", default=0.10, min=0.0, max=1.0, step=0.01, advanced=True), + + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + io.Latent.Output(display_name="latent"), + ], + ) + + @classmethod + def execute(cls, positive, negative, latent, noise_augmentation) -> io.NodeOutput: + latent = latent["samples"] + positive = node_helpers.conditioning_set_values(positive, {"concat_latent_image": latent, "noise_augmentation": noise_augmentation}) + negative = node_helpers.conditioning_set_values(negative, {"concat_latent_image": latent, "noise_augmentation": noise_augmentation}) + out_latent = {} + out_latent["samples"] = torch.zeros([latent.shape[0], 32, latent.shape[-3], latent.shape[-2], latent.shape[-1]], device=comfy.model_management.intermediate_device()) + return io.NodeOutput(positive, negative, out_latent) + + +class HunyuanExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + CLIPTextEncodeHunyuanDiT, + TextEncodeHunyuanVideo_ImageToVideo, + EmptyHunyuanLatentVideo, + EmptyHunyuanVideo15Latent, + HunyuanVideo15ImageToVideo, + HunyuanVideo15SuperResolution, + HunyuanVideo15LatentUpscaleWithModel, + LatentUpscaleModelLoader, + HunyuanImageToVideo, + EmptyHunyuanImageLatent, + HunyuanRefinerLatent, + ] + + +async def comfy_entrypoint() -> HunyuanExtension: + return HunyuanExtension() diff --git a/ComfyUI/comfy_extras/nodes_hunyuan3d.py b/ComfyUI/comfy_extras/nodes_hunyuan3d.py new file mode 100644 index 0000000000000000000000000000000000000000..c5fa946ccbbb9627c72518ef88720788367a8898 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_hunyuan3d.py @@ -0,0 +1,505 @@ +import torch +from comfy.ldm.modules.diffusionmodules.mmdit import get_1d_sincos_pos_embed_from_grid_torch +import comfy.model_management +from comfy_extras.nodes_save_3d import pack_variable_mesh_batch +from typing_extensions import override +from comfy_api.latest import ComfyExtension, IO, Types +from comfy_api.latest._util import MESH, VOXEL # only for backward compatibility if someone import it from this file (will be removed later) # noqa + + +class EmptyLatentHunyuan3Dv2(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="EmptyLatentHunyuan3Dv2", + category="model/latent/hunyuan 3d", + inputs=[ + IO.Int.Input("resolution", default=3072, min=1, max=8192), + IO.Int.Input("batch_size", default=1, min=1, max=4096, tooltip="The number of latent images in the batch."), + ], + outputs=[ + IO.Latent.Output(), + ] + ) + + @classmethod + def execute(cls, resolution, batch_size) -> IO.NodeOutput: + latent = torch.zeros([batch_size, 64, resolution], device=comfy.model_management.intermediate_device()) + return IO.NodeOutput({"samples": latent, "type": "hunyuan3dv2"}) + + generate = execute # TODO: remove + + +class Hunyuan3Dv2Conditioning(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="Hunyuan3Dv2Conditioning", + category="model/conditioning/hunyuan 3d", + inputs=[ + IO.ClipVisionOutput.Input("clip_vision_output"), + ], + outputs=[ + IO.Conditioning.Output(display_name="positive"), + IO.Conditioning.Output(display_name="negative"), + ] + ) + + @classmethod + def execute(cls, clip_vision_output) -> IO.NodeOutput: + embeds = clip_vision_output.last_hidden_state + positive = [[embeds, {}]] + negative = [[torch.zeros_like(embeds), {}]] + return IO.NodeOutput(positive, negative) + + encode = execute # TODO: remove + + +class Hunyuan3Dv2ConditioningMultiView(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="Hunyuan3Dv2ConditioningMultiView", + category="model/conditioning/hunyuan 3d", + inputs=[ + IO.ClipVisionOutput.Input("front", optional=True), + IO.ClipVisionOutput.Input("left", optional=True), + IO.ClipVisionOutput.Input("back", optional=True), + IO.ClipVisionOutput.Input("right", optional=True), + ], + outputs=[ + IO.Conditioning.Output(display_name="positive"), + IO.Conditioning.Output(display_name="negative"), + ] + ) + + @classmethod + def execute(cls, front=None, left=None, back=None, right=None) -> IO.NodeOutput: + all_embeds = [front, left, back, right] + out = [] + pos_embeds = None + for i, e in enumerate(all_embeds): + if e is not None: + if pos_embeds is None: + pos_embeds = get_1d_sincos_pos_embed_from_grid_torch(e.last_hidden_state.shape[-1], torch.arange(4)) + out.append(e.last_hidden_state + pos_embeds[i].reshape(1, 1, -1)) + + embeds = torch.cat(out, dim=1) + positive = [[embeds, {}]] + negative = [[torch.zeros_like(embeds), {}]] + return IO.NodeOutput(positive, negative) + + encode = execute # TODO: remove + + +class VAEDecodeHunyuan3D(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="VAEDecodeHunyuan3D", + category="model/latent/hunyuan 3d", + inputs=[ + IO.Latent.Input("samples"), + IO.Vae.Input("vae"), + IO.Int.Input("num_chunks", default=8000, min=1000, max=500000, advanced=True), + IO.Int.Input("octree_resolution", default=256, min=16, max=512, advanced=True), + ], + outputs=[ + IO.Voxel.Output(), + ] + ) + + @classmethod + def execute(cls, vae, samples, num_chunks, octree_resolution) -> IO.NodeOutput: + voxels = Types.VOXEL(vae.decode(samples["samples"], vae_options={"num_chunks": num_chunks, "octree_resolution": octree_resolution})) + return IO.NodeOutput(voxels) + + decode = execute # TODO: remove + + +def voxel_to_mesh(voxels, threshold=0.5, device=None): + if device is None: + device = torch.device("cpu") + voxels = voxels.to(device) + + binary = (voxels > threshold).float() + padded = torch.nn.functional.pad(binary, (1, 1, 1, 1, 1, 1), 'constant', 0) + + D, H, W = binary.shape + + neighbors = torch.tensor([ + [0, 0, 1], + [0, 0, -1], + [0, 1, 0], + [0, -1, 0], + [1, 0, 0], + [-1, 0, 0] + ], device=device) + + z, y, x = torch.meshgrid( + torch.arange(D, device=device), + torch.arange(H, device=device), + torch.arange(W, device=device), + indexing='ij' + ) + voxel_indices = torch.stack([z.flatten(), y.flatten(), x.flatten()], dim=1) + + solid_mask = binary.flatten() > 0 + solid_indices = voxel_indices[solid_mask] + + corner_offsets = [ + torch.tensor([ + [0, 0, 1], [0, 1, 1], [1, 1, 1], [1, 0, 1] + ], device=device), + torch.tensor([ + [0, 0, 0], [1, 0, 0], [1, 1, 0], [0, 1, 0] + ], device=device), + torch.tensor([ + [0, 1, 0], [1, 1, 0], [1, 1, 1], [0, 1, 1] + ], device=device), + torch.tensor([ + [0, 0, 0], [0, 0, 1], [1, 0, 1], [1, 0, 0] + ], device=device), + torch.tensor([ + [1, 0, 1], [1, 1, 1], [1, 1, 0], [1, 0, 0] + ], device=device), + torch.tensor([ + [0, 1, 0], [0, 1, 1], [0, 0, 1], [0, 0, 0] + ], device=device) + ] + + all_vertices = [] + all_indices = [] + + vertex_count = 0 + + for face_idx, offset in enumerate(neighbors): + neighbor_indices = solid_indices + offset + + padded_indices = neighbor_indices + 1 + + is_exposed = padded[ + padded_indices[:, 0], + padded_indices[:, 1], + padded_indices[:, 2] + ] == 0 + + if not is_exposed.any(): + continue + + exposed_indices = solid_indices[is_exposed] + + corners = corner_offsets[face_idx].unsqueeze(0) + + face_vertices = exposed_indices.unsqueeze(1) + corners + + all_vertices.append(face_vertices.reshape(-1, 3)) + + num_faces = exposed_indices.shape[0] + face_indices = torch.arange( + vertex_count, + vertex_count + 4 * num_faces, + device=device + ).reshape(-1, 4) + + all_indices.append(torch.stack([face_indices[:, 0], face_indices[:, 1], face_indices[:, 2]], dim=1)) + all_indices.append(torch.stack([face_indices[:, 0], face_indices[:, 2], face_indices[:, 3]], dim=1)) + + vertex_count += 4 * num_faces + + if len(all_vertices) > 0: + vertices = torch.cat(all_vertices, dim=0) + faces = torch.cat(all_indices, dim=0) + else: + vertices = torch.zeros((1, 3)) + faces = torch.zeros((1, 3)) + + v_min = 0 + v_max = max(voxels.shape) + + vertices = vertices - (v_min + v_max) / 2 + + scale = (v_max - v_min) / 2 + if scale > 0: + vertices = vertices / scale + + vertices = torch.fliplr(vertices) + return vertices, faces + +def voxel_to_mesh_surfnet(voxels, threshold=0.5, device=None): + if device is None: + device = torch.device("cpu") + voxels = voxels.to(device) + + D, H, W = voxels.shape + + padded = torch.nn.functional.pad(voxels, (1, 1, 1, 1, 1, 1), 'constant', 0) + z, y, x = torch.meshgrid( + torch.arange(D, device=device), + torch.arange(H, device=device), + torch.arange(W, device=device), + indexing='ij' + ) + cell_positions = torch.stack([z.flatten(), y.flatten(), x.flatten()], dim=1) + + corner_offsets = torch.tensor([ + [0, 0, 0], [1, 0, 0], [0, 1, 0], [1, 1, 0], + [0, 0, 1], [1, 0, 1], [0, 1, 1], [1, 1, 1] + ], device=device) + + pos = cell_positions.unsqueeze(1) + corner_offsets.unsqueeze(0) + z_idx, y_idx, x_idx = pos.unbind(-1) + corner_values = padded[z_idx, y_idx, x_idx] + + corner_signs = corner_values > threshold + has_inside = torch.any(corner_signs, dim=1) + has_outside = torch.any(~corner_signs, dim=1) + contains_surface = has_inside & has_outside + + active_cells = cell_positions[contains_surface] + active_signs = corner_signs[contains_surface] + active_values = corner_values[contains_surface] + + if active_cells.shape[0] == 0: + return torch.zeros((0, 3), device=device), torch.zeros((0, 3), dtype=torch.long, device=device) + + edges = torch.tensor([ + [0, 1], [0, 2], [0, 4], [1, 3], + [1, 5], [2, 3], [2, 6], [3, 7], + [4, 5], [4, 6], [5, 7], [6, 7] + ], device=device) + + cell_vertices = {} + progress = comfy.utils.ProgressBar(100) + + for edge_idx, (e1, e2) in enumerate(edges): + progress.update(1) + crossing = active_signs[:, e1] != active_signs[:, e2] + if not crossing.any(): + continue + + cell_indices = torch.nonzero(crossing, as_tuple=True)[0] + + v1 = active_values[cell_indices, e1] + v2 = active_values[cell_indices, e2] + + t = torch.zeros_like(v1, device=device) + denom = v2 - v1 + valid = denom != 0 + t[valid] = (threshold - v1[valid]) / denom[valid] + t[~valid] = 0.5 + + p1 = corner_offsets[e1].float() + p2 = corner_offsets[e2].float() + + intersection = p1.unsqueeze(0) + t.unsqueeze(1) * (p2.unsqueeze(0) - p1.unsqueeze(0)) + + for i, point in zip(cell_indices.tolist(), intersection): + if i not in cell_vertices: + cell_vertices[i] = [] + cell_vertices[i].append(point) + + # Calculate the final vertices as the average of intersection points for each cell + vertices = [] + vertex_lookup = {} + + vert_progress_mod = round(len(cell_vertices)/50) + + for i, points in cell_vertices.items(): + if not i % vert_progress_mod: + progress.update(1) + + if points: + vertex = torch.stack(points).mean(dim=0) + vertex = vertex + active_cells[i].float() + vertex_lookup[tuple(active_cells[i].tolist())] = len(vertices) + vertices.append(vertex) + + if not vertices: + return torch.zeros((0, 3), device=device), torch.zeros((0, 3), dtype=torch.long, device=device) + + final_vertices = torch.stack(vertices) + + inside_corners_mask = active_signs + outside_corners_mask = ~active_signs + + inside_counts = inside_corners_mask.sum(dim=1, keepdim=True).float() + outside_counts = outside_corners_mask.sum(dim=1, keepdim=True).float() + + inside_pos = torch.zeros((active_cells.shape[0], 3), device=device) + outside_pos = torch.zeros((active_cells.shape[0], 3), device=device) + + for i in range(8): + mask_inside = inside_corners_mask[:, i].unsqueeze(1) + mask_outside = outside_corners_mask[:, i].unsqueeze(1) + inside_pos += corner_offsets[i].float().unsqueeze(0) * mask_inside + outside_pos += corner_offsets[i].float().unsqueeze(0) * mask_outside + + inside_pos /= inside_counts + outside_pos /= outside_counts + gradients = inside_pos - outside_pos + + pos_dirs = torch.tensor([ + [1, 0, 0], + [0, 1, 0], + [0, 0, 1] + ], device=device) + + cross_products = [ + torch.linalg.cross(pos_dirs[i].float(), pos_dirs[j].float()) + for i in range(3) for j in range(i+1, 3) + ] + + faces = [] + all_keys = set(vertex_lookup.keys()) + + face_progress_mod = round(len(active_cells)/38*3) + + for pair_idx, (i, j) in enumerate([(0,1), (0,2), (1,2)]): + dir_i = pos_dirs[i] + dir_j = pos_dirs[j] + cross_product = cross_products[pair_idx] + + ni_positions = active_cells + dir_i + nj_positions = active_cells + dir_j + diag_positions = active_cells + dir_i + dir_j + + alignments = torch.matmul(gradients, cross_product) + + valid_quads = [] + quad_indices = [] + + for idx, active_cell in enumerate(active_cells): + if not idx % face_progress_mod: + progress.update(1) + cell_key = tuple(active_cell.tolist()) + ni_key = tuple(ni_positions[idx].tolist()) + nj_key = tuple(nj_positions[idx].tolist()) + diag_key = tuple(diag_positions[idx].tolist()) + + if cell_key in all_keys and ni_key in all_keys and nj_key in all_keys and diag_key in all_keys: + v0 = vertex_lookup[cell_key] + v1 = vertex_lookup[ni_key] + v2 = vertex_lookup[nj_key] + v3 = vertex_lookup[diag_key] + + valid_quads.append((v0, v1, v2, v3)) + quad_indices.append(idx) + + for q_idx, (v0, v1, v2, v3) in enumerate(valid_quads): + cell_idx = quad_indices[q_idx] + if alignments[cell_idx] > 0: + faces.append(torch.tensor([v0, v1, v3], device=device, dtype=torch.long)) + faces.append(torch.tensor([v0, v3, v2], device=device, dtype=torch.long)) + else: + faces.append(torch.tensor([v0, v3, v1], device=device, dtype=torch.long)) + faces.append(torch.tensor([v0, v2, v3], device=device, dtype=torch.long)) + + if faces: + faces = torch.stack(faces) + else: + faces = torch.zeros((0, 3), dtype=torch.long, device=device) + + v_min = 0 + v_max = max(D, H, W) + + final_vertices = final_vertices - (v_min + v_max) / 2 + + scale = (v_max - v_min) / 2 + if scale > 0: + final_vertices = final_vertices / scale + + final_vertices = torch.fliplr(final_vertices) + + return final_vertices, faces + + +class VoxelToMeshBasic(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="VoxelToMeshBasic", + display_name="Voxel to Mesh (Basic) (DEPRECATED)", + category="3d", + description="Converts a voxel grid to a mesh.", + is_deprecated=True, # This node is superseded by the Voxel To Mesh node + inputs=[ + IO.Voxel.Input("voxel"), + IO.Float.Input("threshold", default=0.6, min=-1.0, max=1.0, step=0.01), + ], + outputs=[ + IO.Mesh.Output(), + ], + ) + + @classmethod + def execute(cls, voxel, threshold) -> IO.NodeOutput: + vertices = [] + faces = [] + for x in voxel.data: + v, f = voxel_to_mesh(x, threshold=threshold, device=None) + vertices.append(v) + faces.append(f) + + if vertices and all(v.shape == vertices[0].shape for v in vertices) and all(f.shape == faces[0].shape for f in faces): + return IO.NodeOutput(Types.MESH(torch.stack(vertices), torch.stack(faces))) + return IO.NodeOutput(pack_variable_mesh_batch(vertices, faces)) + + decode = execute # TODO: remove + + +class VoxelToMesh(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="VoxelToMesh", + display_name="Voxel to Mesh", + category="3d", + description="Converts a voxel grid to a mesh.", + inputs=[ + IO.Voxel.Input("voxel"), + IO.Combo.Input("algorithm", options=["surface net", "basic"]), + IO.Float.Input("threshold", default=0.6, min=-1.0, max=1.0, step=0.01), + ], + outputs=[ + IO.Mesh.Output(), + ] + ) + + @classmethod + def execute(cls, voxel, algorithm, threshold) -> IO.NodeOutput: + vertices = [] + faces = [] + + if algorithm == "basic": + mesh_function = voxel_to_mesh + elif algorithm == "surface net": + mesh_function = voxel_to_mesh_surfnet + + for x in voxel.data: + v, f = mesh_function(x, threshold=threshold, device=None) + vertices.append(v) + faces.append(f) + + if vertices and all(v.shape == vertices[0].shape for v in vertices) and all(f.shape == faces[0].shape for f in faces): + return IO.NodeOutput(Types.MESH(torch.stack(vertices), torch.stack(faces))) + return IO.NodeOutput(pack_variable_mesh_batch(vertices, faces)) + + decode = execute # TODO: remove + + +class Hunyuan3dExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[IO.ComfyNode]]: + return [ + EmptyLatentHunyuan3Dv2, + Hunyuan3Dv2Conditioning, + Hunyuan3Dv2ConditioningMultiView, + VAEDecodeHunyuan3D, + VoxelToMeshBasic, + VoxelToMesh, + ] + + +async def comfy_entrypoint() -> Hunyuan3dExtension: + return Hunyuan3dExtension() diff --git a/ComfyUI/comfy_extras/nodes_hypernetwork.py b/ComfyUI/comfy_extras/nodes_hypernetwork.py new file mode 100644 index 0000000000000000000000000000000000000000..2d3f1bd051ce1a5ecee163af0db44e8c45961b6d --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_hypernetwork.py @@ -0,0 +1,139 @@ +import comfy.utils +import folder_paths +import torch +import logging +from comfy_api.latest import IO, ComfyExtension +from typing_extensions import override + + +def load_hypernetwork_patch(path, strength): + sd = comfy.utils.load_torch_file(path, safe_load=True) + activation_func = sd.get('activation_func', 'linear') + is_layer_norm = sd.get('is_layer_norm', False) + use_dropout = sd.get('use_dropout', False) + activate_output = sd.get('activate_output', False) + last_layer_dropout = sd.get('last_layer_dropout', False) + + valid_activation = { + "linear": torch.nn.Identity, + "relu": torch.nn.ReLU, + "leakyrelu": torch.nn.LeakyReLU, + "elu": torch.nn.ELU, + "swish": torch.nn.Hardswish, + "tanh": torch.nn.Tanh, + "sigmoid": torch.nn.Sigmoid, + "softsign": torch.nn.Softsign, + "mish": torch.nn.Mish, + } + + if activation_func not in valid_activation: + logging.error("Unsupported Hypernetwork format, if you report it I might implement it. {} {} {} {} {} {}".format(path, activation_func, is_layer_norm, use_dropout, activate_output, last_layer_dropout)) + return None + + out = {} + + for d in sd: + try: + dim = int(d) + except: + continue + + output = [] + for index in [0, 1]: + attn_weights = sd[dim][index] + keys = attn_weights.keys() + + linears = filter(lambda a: a.endswith(".weight"), keys) + linears = list(map(lambda a: a[:-len(".weight")], linears)) + layers = [] + + i = 0 + while i < len(linears): + lin_name = linears[i] + last_layer = (i == (len(linears) - 1)) + penultimate_layer = (i == (len(linears) - 2)) + + lin_weight = attn_weights['{}.weight'.format(lin_name)] + lin_bias = attn_weights['{}.bias'.format(lin_name)] + layer = torch.nn.Linear(lin_weight.shape[1], lin_weight.shape[0]) + layer.load_state_dict({"weight": lin_weight, "bias": lin_bias}) + layers.append(layer) + if activation_func != "linear": + if (not last_layer) or (activate_output): + layers.append(valid_activation[activation_func]()) + if is_layer_norm: + i += 1 + ln_name = linears[i] + ln_weight = attn_weights['{}.weight'.format(ln_name)] + ln_bias = attn_weights['{}.bias'.format(ln_name)] + ln = torch.nn.LayerNorm(ln_weight.shape[0]) + ln.load_state_dict({"weight": ln_weight, "bias": ln_bias}) + layers.append(ln) + if use_dropout: + if (not last_layer) and (not penultimate_layer or last_layer_dropout): + layers.append(torch.nn.Dropout(p=0.3)) + i += 1 + + output.append(torch.nn.Sequential(*layers)) + out[dim] = torch.nn.ModuleList(output) + + class hypernetwork_patch: + def __init__(self, hypernet, strength): + self.hypernet = hypernet + self.strength = strength + def __call__(self, q, k, v, extra_options): + dim = k.shape[-1] + if dim in self.hypernet: + hn = self.hypernet[dim] + k = k + hn[0](k) * self.strength + v = v + hn[1](v) * self.strength + + return q, k, v + + def to(self, device): + for d in self.hypernet.keys(): + self.hypernet[d] = self.hypernet[d].to(device) + return self + + return hypernetwork_patch(out, strength) + +class HypernetworkLoader(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="HypernetworkLoader", + display_name="Load Hypernetwork", + category="model/loaders", + inputs=[ + IO.Model.Input("model"), + IO.Combo.Input("hypernetwork_name", options=folder_paths.get_filename_list("hypernetworks")), + IO.Float.Input("strength", default=1.0, min=-10.0, max=10.0, step=0.01), + ], + outputs=[ + IO.Model.Output(), + ], + ) + + @classmethod + def execute(cls, model, hypernetwork_name, strength) -> IO.NodeOutput: + hypernetwork_path = folder_paths.get_full_path_or_raise("hypernetworks", hypernetwork_name) + model_hypernetwork = model.clone() + patch = load_hypernetwork_patch(hypernetwork_path, strength) + if patch is not None: + model_hypernetwork.set_model_attn1_patch(patch) + model_hypernetwork.set_model_attn2_patch(patch) + return IO.NodeOutput(model_hypernetwork) + + load_hypernetwork = execute # TODO: remove + + +class HyperNetworkExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[IO.ComfyNode]]: + return [ + HypernetworkLoader, + ] + + +async def comfy_entrypoint() -> HyperNetworkExtension: + return HyperNetworkExtension() diff --git a/ComfyUI/comfy_extras/nodes_hypertile.py b/ComfyUI/comfy_extras/nodes_hypertile.py new file mode 100644 index 0000000000000000000000000000000000000000..2a96416be5c227fb836cb460b74db459a2d7a87c --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_hypertile.py @@ -0,0 +1,98 @@ +#Taken from: https://github.com/tfernd/HyperTile/ + +import math +from typing_extensions import override +from einops import rearrange +# Use torch rng for consistency across generations +from torch import randint +from comfy_api.latest import ComfyExtension, io + +def random_divisor(value: int, min_value: int, /, max_options: int = 1) -> int: + min_value = min(min_value, value) + + # All big divisors of value (inclusive) + divisors = [i for i in range(min_value, value + 1) if value % i == 0] + + ns = [value // i for i in divisors[:max_options]] # has at least 1 element + + if len(ns) - 1 > 0: + idx = randint(low=0, high=len(ns) - 1, size=(1,)).item() + else: + idx = 0 + + return ns[idx] + +class HyperTile(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="HyperTile", + category="model/patch/unet", + inputs=[ + io.Model.Input("model"), + io.Int.Input("tile_size", default=256, min=1, max=2048, advanced=True), + io.Int.Input("swap_size", default=2, min=1, max=128, advanced=True), + io.Int.Input("max_depth", default=0, min=0, max=10, advanced=True), + io.Boolean.Input("scale_depth", default=False, advanced=True), + ], + outputs=[ + io.Model.Output(), + ], + ) + + @classmethod + def execute(cls, model, tile_size, swap_size, max_depth, scale_depth) -> io.NodeOutput: + latent_tile_size = max(32, tile_size) // 8 + temp = None + + def hypertile_in(q, k, v, extra_options): + nonlocal temp + model_chans = q.shape[-2] + orig_shape = extra_options['original_shape'] + apply_to = [] + for i in range(max_depth + 1): + apply_to.append((orig_shape[-2] / (2 ** i)) * (orig_shape[-1] / (2 ** i))) + + if model_chans in apply_to: + shape = extra_options["original_shape"] + aspect_ratio = shape[-1] / shape[-2] + + hw = q.size(1) + h, w = round(math.sqrt(hw * aspect_ratio)), round(math.sqrt(hw / aspect_ratio)) + + factor = (2 ** apply_to.index(model_chans)) if scale_depth else 1 + nh = random_divisor(h, latent_tile_size * factor, swap_size) + nw = random_divisor(w, latent_tile_size * factor, swap_size) + + if nh * nw > 1: + q = rearrange(q, "b (nh h nw w) c -> (b nh nw) (h w) c", h=h // nh, w=w // nw, nh=nh, nw=nw) + temp = (nh, nw, h, w) + return q, k, v + + return q, k, v + def hypertile_out(out, extra_options): + nonlocal temp + if temp is not None: + nh, nw, h, w = temp + temp = None + out = rearrange(out, "(b nh nw) hw c -> b nh nw hw c", nh=nh, nw=nw) + out = rearrange(out, "b nh nw (h w) c -> b (nh h nw w) c", h=h // nh, w=w // nw) + return out + + + m = model.clone() + m.set_model_attn1_patch(hypertile_in) + m.set_model_attn1_output_patch(hypertile_out) + return (m, ) + + +class HyperTileExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + HyperTile, + ] + + +async def comfy_entrypoint() -> HyperTileExtension: + return HyperTileExtension() diff --git a/ComfyUI/comfy_extras/nodes_ideogram4.py b/ComfyUI/comfy_extras/nodes_ideogram4.py new file mode 100644 index 0000000000000000000000000000000000000000..4070db17c2bbf91c197b4eb5bab1189566107597 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_ideogram4.py @@ -0,0 +1,64 @@ +"""Ideogram 4 sampling helper +""" + +import math + +import torch +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io + +_LOGSNR_MIN = -15.0 +_LOGSNR_MAX = 18.0 + + +def _logit_normal_schedule(u, mean, std): + # Reference time (0=noise..1=clean) via the probit/ndtri quantile. + u = torch.as_tensor(u, dtype=torch.float64) + t = 1.0 - torch.special.expit(mean + std * torch.special.ndtri(u)) + t_min = 1.0 / (1.0 + math.exp(0.5 * _LOGSNR_MAX)) + t_max = 1.0 / (1.0 + math.exp(0.5 * _LOGSNR_MIN)) + return t.clamp(t_min, t_max) + + +def ideogram4_sigmas(num_steps, width, height, mu, std): + """Descending sigmas (len num_steps+1) for the reference schedule. + + mu + the resolution term form the logSNR shift; std is the spread. + """ + mean = mu + 0.5 * math.log((width * height) / (512 * 512)) + u = torch.linspace(0.0, 1.0, num_steps + 1, dtype=torch.float64) + sigmas = (1.0 - _logit_normal_schedule(u, mean, std)).flip(0) + sigmas[-1] = 0.0 # clamp leaves ~6e-4; force full denoise + return sigmas.to(torch.float32) + + +class Ideogram4Scheduler(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="Ideogram4Scheduler", + display_name="Ideogram 4 Scheduler", + category="model/sampling/schedulers", + inputs=[ + io.Int.Input("steps", default=20, min=1, max=200), + io.Int.Input("width", default=1024, min=256, max=8192, step=16), + io.Int.Input("height", default=1024, min=256, max=8192, step=16), + io.Float.Input("mu", default=0.0, min=-10.0, max=10.0, step=0.05), + io.Float.Input("std", default=1.75, min=0.1, max=5.0, step=0.05), + ], + outputs=[io.Sigmas.Output()], + ) + + @classmethod + def execute(cls, steps, width, height, mu, std) -> io.NodeOutput: + return io.NodeOutput(ideogram4_sigmas(steps, width, height, mu, std)) + + +class Ideogram4Extension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [Ideogram4Scheduler] + + +async def comfy_entrypoint() -> Ideogram4Extension: + return Ideogram4Extension() diff --git a/ComfyUI/comfy_extras/nodes_image_compare.py b/ComfyUI/comfy_extras/nodes_image_compare.py new file mode 100644 index 0000000000000000000000000000000000000000..58af9ae82df44e534d172e15014ba02841232910 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_image_compare.py @@ -0,0 +1,54 @@ +import nodes + +from typing_extensions import override +from comfy_api.latest import IO, ComfyExtension + + +class ImageCompare(IO.ComfyNode): + """Compares two images with a slider interface.""" + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="ImageCompare", + display_name="Compare Images", + description="Compares two images side by side with a slider.", + category="image", + essentials_category="Image Tools", + is_experimental=True, + is_output_node=True, + inputs=[ + IO.Image.Input("image_a", optional=True), + IO.Image.Input("image_b", optional=True), + IO.ImageCompare.Input("compare_view"), + ], + outputs=[], + ) + + @classmethod + def execute(cls, image_a=None, image_b=None, compare_view=None) -> IO.NodeOutput: + result = {"a_images": [], "b_images": []} + + preview_node = nodes.PreviewImage() + + if image_a is not None and len(image_a) > 0: + saved = preview_node.save_images(image_a, "comfy.compare.a") + result["a_images"] = saved["ui"]["images"] + + if image_b is not None and len(image_b) > 0: + saved = preview_node.save_images(image_b, "comfy.compare.b") + result["b_images"] = saved["ui"]["images"] + + return IO.NodeOutput(ui=result) + + +class ImageCompareExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[IO.ComfyNode]]: + return [ + ImageCompare, + ] + + +async def comfy_entrypoint() -> ImageCompareExtension: + return ImageCompareExtension() diff --git a/ComfyUI/comfy_extras/nodes_images.py b/ComfyUI/comfy_extras/nodes_images.py new file mode 100644 index 0000000000000000000000000000000000000000..7011d9c13c463ac3020fb8d08f24374c361d3509 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_images.py @@ -0,0 +1,1267 @@ +import nodes +import folder_paths + +import av +import json + +import os +import re +import math +import numpy as np +import struct +import torch + +import zlib +import comfy.utils +from fractions import Fraction + +from server import PromptServer +from comfy_api.latest import ComfyExtension, IO, UI +from comfy.cli_args import args +from typing_extensions import override + +SVG = IO.SVG.Type # TODO: temporary solution for backward compatibility, will be removed later. + +MAX_RESOLUTION = nodes.MAX_RESOLUTION + +class ImageCrop(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="ImageCrop", + search_aliases=["trim"], + display_name="Crop Image (DEPRECATED)", + category="image/transform", + is_deprecated=True, + essentials_category="Image Tools", + inputs=[ + IO.Image.Input("image"), + IO.Int.Input("width", default=512, min=1, max=nodes.MAX_RESOLUTION, step=1), + IO.Int.Input("height", default=512, min=1, max=nodes.MAX_RESOLUTION, step=1), + IO.Int.Input("x", default=0, min=0, max=nodes.MAX_RESOLUTION, step=1), + IO.Int.Input("y", default=0, min=0, max=nodes.MAX_RESOLUTION, step=1), + ], + outputs=[IO.Image.Output()], + ) + + @classmethod + def execute(cls, image, width, height, x, y) -> IO.NodeOutput: + x = min(x, image.shape[2] - 1) + y = min(y, image.shape[1] - 1) + to_x = width + x + to_y = height + y + img = image[:,y:to_y, x:to_x, :] + return IO.NodeOutput(img) + + crop = execute # TODO: remove + + +class ImageCropV2(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="ImageCropV2", + search_aliases=["crop", "cut", "trim"], + display_name="Crop Image", + category="image/transform", + description = "Crop an image to the specified dimensions.", + essentials_category="Image Tools", + has_intermediate_output=True, + inputs=[ + IO.Image.Input("image"), + IO.BoundingBox.Input("crop_region", component="ImageCrop"), + ], + outputs=[IO.Image.Output()], + ) + + @classmethod + def execute(cls, image, crop_region) -> IO.NodeOutput: + x = crop_region.get("x", 0) + y = crop_region.get("y", 0) + width = crop_region.get("width", 512) + height = crop_region.get("height", 512) + + x = min(x, image.shape[2] - 1) + y = min(y, image.shape[1] - 1) + to_x = width + x + to_y = height + y + img = image[:,y:to_y, x:to_x, :] + return IO.NodeOutput(img, ui=UI.PreviewImage(img)) + + +class BoundingBox(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="PrimitiveBoundingBox", + display_name="Bounding Box", + category="utilities/primitive", + inputs=[ + IO.Int.Input("x", default=0, min=0, max=MAX_RESOLUTION), + IO.Int.Input("y", default=0, min=0, max=MAX_RESOLUTION), + IO.Int.Input("width", default=512, min=1, max=MAX_RESOLUTION), + IO.Int.Input("height", default=512, min=1, max=MAX_RESOLUTION), + ], + outputs=[IO.BoundingBox.Output()], + ) + + @classmethod + def execute(cls, x, y, width, height) -> IO.NodeOutput: + return IO.NodeOutput({"x": x, "y": y, "width": width, "height": height}) + + +class RepeatImageBatch(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="RepeatImageBatch", + search_aliases=["duplicate image", "clone image"], + display_name="Repeat Image Batch", + category="image/batch", + inputs=[ + IO.Image.Input("image"), + IO.Int.Input("amount", default=1, min=1, max=4096), + ], + outputs=[IO.Image.Output()], + ) + + @classmethod + def execute(cls, image, amount) -> IO.NodeOutput: + s = image.repeat((amount, 1,1,1)) + return IO.NodeOutput(s) + + repeat = execute # TODO: remove + + +class ImageFromBatch(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="ImageFromBatch", + search_aliases=["select image", "pick from batch", "extract image"], + display_name="Get Image from Batch", + category="image/batch", + inputs=[ + IO.Image.Input("image"), + IO.Int.Input("batch_index", default=0, min=-MAX_RESOLUTION, max=MAX_RESOLUTION), + IO.Int.Input("length", default=1, min=1, max=4096), + ], + outputs=[IO.Image.Output()], + ) + + @classmethod + def execute(cls, image, batch_index, length) -> IO.NodeOutput: + s_in = image + if batch_index < 0: + batch_index += s_in.shape[0] + batch_index = max(0, min(s_in.shape[0] - 1, batch_index)) + length = min(s_in.shape[0] - batch_index, length) + s = s_in[batch_index:batch_index + length].clone() + return IO.NodeOutput(s) + + frombatch = execute # TODO: remove + + +class ImageAddNoise(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="ImageAddNoise", + search_aliases=["film grain"], + display_name="Add Noise to Image", + category="image/filters", + inputs=[ + IO.Image.Input("image"), + IO.Int.Input( + "seed", + default=0, + min=0, + max=0xFFFFFFFFFFFFFFFF, + control_after_generate=True, + tooltip="The random seed used for creating the noise.", + ), + IO.Float.Input("strength", default=0.5, min=0.0, max=1.0, step=0.01), + ], + outputs=[IO.Image.Output()], + ) + + @classmethod + def execute(cls, image, seed, strength) -> IO.NodeOutput: + generator = torch.manual_seed(seed) + s = torch.clip((image + strength * torch.randn(image.size(), generator=generator, device="cpu").to(image)), min=0.0, max=1.0) + return IO.NodeOutput(s) + + repeat = execute # TODO: remove + + +class SaveAnimatedWEBP(IO.ComfyNode): + COMPRESS_METHODS = {"default": 4, "fastest": 0, "slowest": 6} + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="SaveAnimatedWEBP", + display_name="Save Animated WEBP", + category="image", + inputs=[ + IO.Image.Input("images"), + IO.String.Input("filename_prefix", default="ComfyUI"), + IO.Float.Input("fps", default=6.0, min=0.01, max=1000.0, step=0.01), + IO.Boolean.Input("lossless", default=True), + IO.Int.Input("quality", default=80, min=0, max=100), + IO.Combo.Input("method", options=list(cls.COMPRESS_METHODS.keys())), + # "num_frames": ("INT", {"default": 0, "min": 0, "max": 8192}), + ], + hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], + is_output_node=True, + outputs=[IO.Image.Output(display_name="images")] + ) + + @classmethod + def execute(cls, images, fps, filename_prefix, lossless, quality, method, num_frames=0) -> IO.NodeOutput: + return IO.NodeOutput( + images, + ui=UI.ImageSaveHelper.get_save_animated_webp_ui( + images=images, + filename_prefix=filename_prefix, + cls=cls, + fps=fps, + lossless=lossless, + quality=quality, + method=cls.COMPRESS_METHODS.get(method) + ) + ) + + +class SaveAnimatedPNG(IO.ComfyNode): + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="SaveAnimatedPNG", + display_name="Save Animated PNG", + category="image", + inputs=[ + IO.Image.Input("images"), + IO.String.Input("filename_prefix", default="ComfyUI"), + IO.Float.Input("fps", default=6.0, min=0.01, max=1000.0, step=0.01), + IO.Int.Input("compress_level", default=4, min=0, max=9, advanced=True), + ], + hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], + is_output_node=True, + outputs=[IO.Image.Output(display_name="images")] + ) + + @classmethod + def execute(cls, images, fps, compress_level, filename_prefix="ComfyUI") -> IO.NodeOutput: + return IO.NodeOutput( + images, + ui=UI.ImageSaveHelper.get_save_animated_png_ui( + images=images, + filename_prefix=filename_prefix, + cls=cls, + fps=fps, + compress_level=compress_level, + ) + ) + + +class ImageStitch(IO.ComfyNode): + """Upstreamed from https://github.com/kijai/ComfyUI-KJNodes""" + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="ImageStitch", + search_aliases=["combine images", "join images", "concatenate images", "side by side"], + display_name="Stitch Images", + description="Stitches image2 to image1 in the specified direction.\n" + "If image2 is not provided, returns image1 unchanged.\n" + "Optional spacing can be added between images.", + category="image/transform", + inputs=[ + IO.Image.Input("image1"), + IO.Combo.Input("direction", options=["right", "down", "left", "up"], default="right"), + IO.Boolean.Input("match_image_size", default=True), + IO.Int.Input("spacing_width", default=0, min=0, max=1024, step=2, advanced=True), + IO.Combo.Input("spacing_color", options=["white", "black", "red", "green", "blue"], default="white", advanced=True), + IO.Image.Input("image2", optional=True), + ], + outputs=[IO.Image.Output()], + ) + + @classmethod + def execute( + cls, + image1, + direction, + match_image_size, + spacing_width, + spacing_color, + image2=None, + ) -> IO.NodeOutput: + if image2 is None: + return IO.NodeOutput(image1) + + # Handle batch size differences + if image1.shape[0] != image2.shape[0]: + max_batch = max(image1.shape[0], image2.shape[0]) + if image1.shape[0] < max_batch: + image1 = torch.cat( + [image1, image1[-1:].repeat(max_batch - image1.shape[0], 1, 1, 1)] + ) + if image2.shape[0] < max_batch: + image2 = torch.cat( + [image2, image2[-1:].repeat(max_batch - image2.shape[0], 1, 1, 1)] + ) + + # Match image sizes if requested + if match_image_size: + h1, w1 = image1.shape[1:3] + h2, w2 = image2.shape[1:3] + aspect_ratio = w2 / h2 + + if direction in ["left", "right"]: + target_h, target_w = h1, int(h1 * aspect_ratio) + else: # up, down + target_w, target_h = w1, int(w1 / aspect_ratio) + + image2 = comfy.utils.common_upscale( + image2.movedim(-1, 1), target_w, target_h, "lanczos", "disabled" + ).movedim(1, -1) + + color_map = { + "white": 1.0, + "black": 0.0, + "red": (1.0, 0.0, 0.0), + "green": (0.0, 1.0, 0.0), + "blue": (0.0, 0.0, 1.0), + } + + color_val = color_map[spacing_color] + + # When not matching sizes, pad to align non-concat dimensions + if not match_image_size: + h1, w1 = image1.shape[1:3] + h2, w2 = image2.shape[1:3] + pad_value = 0.0 + if not isinstance(color_val, tuple): + pad_value = color_val + + if direction in ["left", "right"]: + # For horizontal concat, pad heights to match + if h1 != h2: + target_h = max(h1, h2) + if h1 < target_h: + pad_h = target_h - h1 + pad_top, pad_bottom = pad_h // 2, pad_h - pad_h // 2 + image1 = torch.nn.functional.pad(image1, (0, 0, 0, 0, pad_top, pad_bottom), mode='constant', value=pad_value) + if h2 < target_h: + pad_h = target_h - h2 + pad_top, pad_bottom = pad_h // 2, pad_h - pad_h // 2 + image2 = torch.nn.functional.pad(image2, (0, 0, 0, 0, pad_top, pad_bottom), mode='constant', value=pad_value) + else: # up, down + # For vertical concat, pad widths to match + if w1 != w2: + target_w = max(w1, w2) + if w1 < target_w: + pad_w = target_w - w1 + pad_left, pad_right = pad_w // 2, pad_w - pad_w // 2 + image1 = torch.nn.functional.pad(image1, (0, 0, pad_left, pad_right), mode='constant', value=pad_value) + if w2 < target_w: + pad_w = target_w - w2 + pad_left, pad_right = pad_w // 2, pad_w - pad_w // 2 + image2 = torch.nn.functional.pad(image2, (0, 0, pad_left, pad_right), mode='constant', value=pad_value) + + # Ensure same number of channels + if image1.shape[-1] != image2.shape[-1]: + max_channels = max(image1.shape[-1], image2.shape[-1]) + if image1.shape[-1] < max_channels: + image1 = torch.cat( + [ + image1, + torch.ones( + *image1.shape[:-1], + max_channels - image1.shape[-1], + device=image1.device, + ), + ], + dim=-1, + ) + if image2.shape[-1] < max_channels: + image2 = torch.cat( + [ + image2, + torch.ones( + *image2.shape[:-1], + max_channels - image2.shape[-1], + device=image2.device, + ), + ], + dim=-1, + ) + + # Add spacing if specified + if spacing_width > 0: + spacing_width = spacing_width + (spacing_width % 2) # Ensure even + + if direction in ["left", "right"]: + spacing_shape = ( + image1.shape[0], + max(image1.shape[1], image2.shape[1]), + spacing_width, + image1.shape[-1], + ) + else: + spacing_shape = ( + image1.shape[0], + spacing_width, + max(image1.shape[2], image2.shape[2]), + image1.shape[-1], + ) + + spacing = torch.full(spacing_shape, 0.0, device=image1.device) + if isinstance(color_val, tuple): + for i, c in enumerate(color_val): + if i < spacing.shape[-1]: + spacing[..., i] = c + if spacing.shape[-1] == 4: # Add alpha + spacing[..., 3] = 1.0 + else: + spacing[..., : min(3, spacing.shape[-1])] = color_val + if spacing.shape[-1] == 4: + spacing[..., 3] = 1.0 + + # Concatenate images + images = [image2, image1] if direction in ["left", "up"] else [image1, image2] + if spacing_width > 0: + images.insert(1, spacing) + + concat_dim = 2 if direction in ["left", "right"] else 1 + return IO.NodeOutput(torch.cat(images, dim=concat_dim)) + + stitch = execute # TODO: remove + + +class ResizeAndPadImage(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="ResizeAndPadImage", + search_aliases=["fit to size"], + display_name="Resize And Pad Image", + category="image/transform", + inputs=[ + IO.Image.Input("image"), + IO.Int.Input("target_width", default=512, min=1, max=nodes.MAX_RESOLUTION, step=1), + IO.Int.Input("target_height", default=512, min=1, max=nodes.MAX_RESOLUTION, step=1), + IO.Combo.Input("padding_color", options=["white", "black"], advanced=True), + IO.Combo.Input("interpolation", options=["area", "bicubic", "nearest-exact", "bilinear", "lanczos"], advanced=True), + ], + outputs=[IO.Image.Output()], + ) + + @classmethod + def execute(cls, image, target_width, target_height, padding_color, interpolation) -> IO.NodeOutput: + batch_size, orig_height, orig_width, channels = image.shape + + scale_w = target_width / orig_width + scale_h = target_height / orig_height + scale = min(scale_w, scale_h) + + new_width = int(orig_width * scale) + new_height = int(orig_height * scale) + + image_permuted = image.permute(0, 3, 1, 2) + + resized = comfy.utils.common_upscale(image_permuted, new_width, new_height, interpolation, "disabled") + + pad_value = 0.0 if padding_color == "black" else 1.0 + padded = torch.full( + (batch_size, channels, target_height, target_width), + pad_value, + dtype=image.dtype, + device=image.device + ) + + y_offset = (target_height - new_height) // 2 + x_offset = (target_width - new_width) // 2 + + padded[:, :, y_offset:y_offset + new_height, x_offset:x_offset + new_width] = resized + + output = padded.permute(0, 2, 3, 1) + return IO.NodeOutput(output) + + resize_and_pad = execute # TODO: remove + + +class SaveSVGNode(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="SaveSVGNode", + search_aliases=["export vector", "save vector graphics"], + display_name="Save SVG", + description="Save SVG files on disk.", + category="image", + inputs=[ + IO.SVG.Input("svg"), + IO.String.Input( + "filename_prefix", + default="svg/ComfyUI", + tooltip="The prefix for the file to save. This may include formatting information such as %date:yyyy-MM-dd% or %Empty Latent Image.width% to include values from nodes.", + ), + ], + hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], + is_output_node=True, + outputs=[IO.SVG.Output("svg")], + ) + + @classmethod + def execute(cls, svg: IO.SVG.Type, filename_prefix="svg/ComfyUI") -> IO.NodeOutput: + full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, folder_paths.get_output_directory()) + results: list[UI.SavedResult] = [] + + # Prepare metadata JSON + metadata_dict = {} + if cls.hidden.prompt is not None: + metadata_dict["prompt"] = cls.hidden.prompt + if cls.hidden.extra_pnginfo is not None: + metadata_dict.update(cls.hidden.extra_pnginfo) + + # Convert metadata to JSON string + metadata_json = json.dumps(metadata_dict, indent=2) if metadata_dict else None + + + for batch_number, svg_bytes in enumerate(svg.data): + filename_with_batch_num = filename.replace("%batch_num%", str(batch_number)) + file = f"{filename_with_batch_num}_{counter:05}_.svg" + + # Read SVG content + svg_bytes.seek(0) + svg_content = svg_bytes.read().decode('utf-8') + + # Inject metadata if available + if metadata_json: + # Create metadata element with CDATA section + metadata_element = f""" + + + """ + # Insert metadata after opening svg tag using regex with a replacement function + def replacement(match): + # match.group(1) contains the captured tag + return match.group(1) + '\n' + metadata_element + + # Apply the substitution + svg_content = re.sub(r'(]*>)', replacement, svg_content, flags=re.UNICODE) + + # Write the modified SVG to file + with open(os.path.join(full_output_folder, file), 'wb') as svg_file: + svg_file.write(svg_content.encode('utf-8')) + + results.append(UI.SavedResult(filename=file, subfolder=subfolder, type=IO.FolderType.output)) + counter += 1 + return IO.NodeOutput(svg, ui={"images": results}) + + +class GetImageSize(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="GetImageSize", + search_aliases=["dimensions", "resolution", "image info"], + display_name="Get Image Size", + description="Returns width and height of the image, and passes it through unchanged.", + category="image", + inputs=[ + IO.Image.Input("image"), + ], + outputs=[ + IO.Int.Output(display_name="width"), + IO.Int.Output(display_name="height"), + IO.Int.Output(display_name="batch_size"), + ], + hidden=[IO.Hidden.unique_id], + ) + + @classmethod + def execute(cls, image) -> IO.NodeOutput: + height = image.shape[1] + width = image.shape[2] + batch_size = image.shape[0] + + # Send progress text to display size on the node + if cls.hidden.unique_id: + PromptServer.instance.send_progress_text(f"width: {width}, height: {height}\n batch size: {batch_size}", cls.hidden.unique_id) + + return IO.NodeOutput(width, height, batch_size) + + get_size = execute # TODO: remove + + +class ImageRotate(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="ImageRotate", + display_name="Rotate Image", + search_aliases=["turn", "flip orientation"], + category="image/transform", + essentials_category="Image Tools", + inputs=[ + IO.Image.Input("image"), + IO.Combo.Input("rotation", options=["none", "90 degrees", "180 degrees", "270 degrees"]), + ], + outputs=[IO.Image.Output()], + ) + + @classmethod + def execute(cls, image, rotation) -> IO.NodeOutput: + rotate_by = 0 + if rotation.startswith("90"): + rotate_by = 1 + elif rotation.startswith("180"): + rotate_by = 2 + elif rotation.startswith("270"): + rotate_by = 3 + + image = torch.rot90(image, k=rotate_by, dims=[2, 1]) + return IO.NodeOutput(image) + + rotate = execute # TODO: remove + + +class ImageFlip(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="ImageFlip", + search_aliases=["mirror", "reflect"], + display_name="Flip Image", + category="image/transform", + inputs=[ + IO.Image.Input("image"), + IO.Combo.Input("flip_method", options=["x-axis: vertically", "y-axis: horizontally"]), + ], + outputs=[IO.Image.Output()], + ) + + @classmethod + def execute(cls, image, flip_method) -> IO.NodeOutput: + if flip_method.startswith("x"): + image = torch.flip(image, dims=[1]) + elif flip_method.startswith("y"): + image = torch.flip(image, dims=[2]) + + return IO.NodeOutput(image) + + flip = execute # TODO: remove + + +class ImageScaleToMaxDimension(IO.ComfyNode): + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="ImageScaleToMaxDimension", + display_name="Scale Image to Max Dimension", + category="image/upscaling", + inputs=[ + IO.Image.Input("image"), + IO.Combo.Input( + "upscale_method", + options=["area", "lanczos", "bilinear", "nearest-exact", "bilinear", "bicubic"], + ), + IO.Int.Input("largest_size", default=512, min=0, max=MAX_RESOLUTION, step=1), + ], + outputs=[IO.Image.Output()], + ) + + @classmethod + def execute(cls, image, upscale_method, largest_size) -> IO.NodeOutput: + height = image.shape[1] + width = image.shape[2] + + if height > width: + width = round((width / height) * largest_size) + height = largest_size + elif width > height: + height = round((height / width) * largest_size) + width = largest_size + else: + height = largest_size + width = largest_size + + samples = image.movedim(-1, 1) + s = comfy.utils.common_upscale(samples, width, height, upscale_method, "disabled") + s = s.movedim(1, -1) + return IO.NodeOutput(s) + + upscale = execute # TODO: remove + + +class SplitImageToTileList(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="SplitImageToTileList", + category="image/batch", + search_aliases=["split image", "tile image", "slice image"], + display_name="Split Image into List of Tiles", + description="Splits an image into a batched list of tiles with a specified overlap.", + inputs=[ + IO.Image.Input("image"), + IO.Int.Input("tile_width", default=1024, min=64, max=MAX_RESOLUTION), + IO.Int.Input("tile_height", default=1024, min=64, max=MAX_RESOLUTION), + IO.Int.Input("overlap", default=128, min=0, max=4096), + ], + outputs=[ + IO.Image.Output(is_output_list=True), + ], + ) + + @staticmethod + def get_grid_coords(width, height, tile_width, tile_height, overlap): + coords = [] + stride_x = round(max(tile_width * 0.25, tile_width - overlap)) + stride_y = round(max(tile_height * 0.25, tile_height - overlap)) + + y = 0 + while y < height: + x = 0 + y_end = min(y + tile_height, height) + y_start = max(0, y_end - tile_height) + + while x < width: + x_end = min(x + tile_width, width) + x_start = max(0, x_end - tile_width) + + coords.append((x_start, y_start, x_end, y_end)) + + if x_end >= width: + break + x += stride_x + + if y_end >= height: + break + y += stride_y + + return coords + + @classmethod + def execute(cls, image, tile_width, tile_height, overlap): + b, h, w, c = image.shape + coords = cls.get_grid_coords(w, h, tile_width, tile_height, overlap) + + output_list = [] + for (x_start, y_start, x_end, y_end) in coords: + tile = image[:, y_start:y_end, x_start:x_end, :] + output_list.append(tile) + + return IO.NodeOutput(output_list) + + +class ImageMergeTileList(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="ImageMergeTileList", + display_name="Merge List of Tiles to Image", + category="image/batch", + search_aliases=["split image", "tile image", "slice image"], + is_input_list=True, + inputs=[ + IO.Image.Input("image_list"), + IO.Int.Input("final_width", default=1024, min=64, max=32768), + IO.Int.Input("final_height", default=1024, min=64, max=32768), + IO.Int.Input("overlap", default=128, min=0, max=4096), + ], + outputs=[ + IO.Image.Output(is_output_list=False), + ], + ) + + @classmethod + def execute(cls, image_list, final_width, final_height, overlap): + w = final_width[0] + h = final_height[0] + ovlp = overlap[0] + feather_str = 1.0 + + first_tile = image_list[0] + b, t_h, t_w, c = first_tile.shape + device = first_tile.device + dtype = first_tile.dtype + + coords = SplitImageToTileList.get_grid_coords(w, h, t_w, t_h, ovlp) + + canvas = torch.zeros((b, h, w, c), device=device, dtype=dtype) + weights = torch.zeros((b, h, w, 1), device=device, dtype=dtype) + + if ovlp > 0: + y_w = torch.sin(math.pi * torch.linspace(0, 1, t_h, device=device, dtype=dtype)) + x_w = torch.sin(math.pi * torch.linspace(0, 1, t_w, device=device, dtype=dtype)) + y_w = torch.clamp(y_w, min=1e-5) + x_w = torch.clamp(x_w, min=1e-5) + + sine_mask = (y_w.unsqueeze(1) * x_w.unsqueeze(0)).unsqueeze(0).unsqueeze(-1) + flat_mask = torch.ones_like(sine_mask) + + weight_mask = torch.lerp(flat_mask, sine_mask, feather_str) + else: + weight_mask = torch.ones((1, t_h, t_w, 1), device=device, dtype=dtype) + + for i, (x_start, y_start, x_end, y_end) in enumerate(coords): + if i >= len(image_list): + break + + tile = image_list[i] + + region_h = y_end - y_start + region_w = x_end - x_start + + real_h = min(region_h, tile.shape[1]) + real_w = min(region_w, tile.shape[2]) + + y_end_actual = y_start + real_h + x_end_actual = x_start + real_w + + tile_crop = tile[:, :real_h, :real_w, :] + mask_crop = weight_mask[:, :real_h, :real_w, :] + + canvas[:, y_start:y_end_actual, x_start:x_end_actual, :] += tile_crop * mask_crop + weights[:, y_start:y_end_actual, x_start:x_end_actual, :] += mask_crop + + weights[weights == 0] = 1.0 + merged_image = canvas / weights + + return IO.NodeOutput(merged_image) + + +# --------------------------------------------------------------------------- +# Format specifications +# --------------------------------------------------------------------------- + +# Maps (file_format, bit_depth, num_channels) -> (quantization scale, numpy dtype, +# av frame pix_fmt, stream pix_fmt). Keeps the encode path declarative instead of branchy. +_FORMAT_SPECS = { + ("png", "8-bit", 1): {"scale": 255.0, "dtype": np.uint8, "frame_fmt": "gray", "stream_fmt": "gray"}, + ("png", "8-bit", 3): {"scale": 255.0, "dtype": np.uint8, "frame_fmt": "rgb24", "stream_fmt": "rgb24"}, + ("png", "8-bit", 4): {"scale": 255.0, "dtype": np.uint8, "frame_fmt": "rgba", "stream_fmt": "rgba"}, + ("png", "16-bit", 1): {"scale": 65535.0, "dtype": np.uint16, "frame_fmt": "gray16le", "stream_fmt": "gray16be"}, + ("png", "16-bit", 3): {"scale": 65535.0, "dtype": np.uint16, "frame_fmt": "rgb48le", "stream_fmt": "rgb48be"}, + ("png", "16-bit", 4): {"scale": 65535.0, "dtype": np.uint16, "frame_fmt": "rgba64le", "stream_fmt": "rgba64be"}, + ("exr", "32-bit float", 1): {"scale": 1.0, "dtype": np.float32, "frame_fmt": "grayf32le", "stream_fmt": "grayf32le"}, + ("exr", "32-bit float", 3): {"scale": 1.0, "dtype": np.float32, "frame_fmt": "gbrpf32le", "stream_fmt": "gbrpf32le"}, + ("exr", "32-bit float", 4): {"scale": 1.0, "dtype": np.float32, "frame_fmt": "gbrapf32le", "stream_fmt": "gbrapf32le"}, +} + + +# --------------------------------------------------------------------------- +# Color transforms +# --------------------------------------------------------------------------- + +def srgb_to_linear(t: torch.Tensor) -> torch.Tensor: + """Inverse sRGB EOTF (IEC 61966-2-1). Operates on RGB channels only; + alpha (if present as the 4th channel) is passed through unchanged.""" + if t.shape[-1] == 4: + rgb, alpha = t[..., :3], t[..., 3:] + return torch.cat([srgb_to_linear(rgb), alpha], dim=-1) + + # Piecewise: linear toe below 0.04045, gamma curve above. + low = t / 12.92 + high = ((t.clamp(min=0.0) + 0.055) / 1.055) ** 2.4 + return torch.where(t <= 0.04045, low, high) + + +# HLG OETF constants from BT.2100 Table 5. +_HLG_A = 0.17883277 +_HLG_B = 0.28466892 +_HLG_C = 0.55991072928 # = 0.5 - a*ln(4*a) + + +def hlg_to_linear(t: torch.Tensor) -> torch.Tensor: + """Inverse HLG OETF (BT.2100). Maps a non-linear HLG signal in [0, 1] to + *scene*-linear light in [0, 1]. Per BT.2100 Note 5a, this is the correct + transform when converting HLG to a linear scene-light representation + (rather than display-light, which would also involve the HLG OOTF). + + Operates on RGB channels only; alpha is passed through unchanged.""" + if t.shape[-1] == 4: + rgb, alpha = t[..., :3], t[..., 3:] + return torch.cat([hlg_to_linear(rgb), alpha], dim=-1) + + # Piecewise: sqrt branch below 0.5, log branch above. + # Clamp the log branch at the 0.5 branch point (not above it) so the + # unselected lane stays finite in exp() without altering selected values; + # values above 1.0 are allowed and extrapolate naturally. + low = (t ** 2) / 3.0 + high = (torch.exp((t.clamp(min=0.5) - _HLG_C) / _HLG_A) + _HLG_B) / 12.0 + return torch.where(t <= 0.5, low, high) + + +# --------------------------------------------------------------------------- +# Metadata injection +# --------------------------------------------------------------------------- + +_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" + + +def _png_chunk(chunk_type: bytes, data: bytes) -> bytes: + """Build a single PNG chunk: length | type | data | CRC32(type+data).""" + crc = zlib.crc32(chunk_type + data) & 0xFFFFFFFF + return struct.pack(">I", len(data)) + chunk_type + data + struct.pack(">I", crc) + + +def _png_text_chunk(keyword: str, text: str) -> bytes: + """tEXt chunk: latin-1 keyword + NUL + latin-1 text.""" + payload = keyword.encode("latin-1") + b"\x00" + text.encode("latin-1", errors="replace") + return _png_chunk(b"tEXt", payload) + + +def inject_png_metadata(png_bytes: bytes, prompt: dict | None, extra_pnginfo: dict | None) -> bytes: + """Insert ComfyUI prompt/workflow as tEXt chunks right after IHDR.""" + if not png_bytes.startswith(_PNG_SIGNATURE): + return png_bytes + + chunks: list[bytes] = [] + if prompt is not None: + chunks.append(_png_text_chunk("prompt", json.dumps(prompt))) + if extra_pnginfo: + for key, value in extra_pnginfo.items(): + chunks.append(_png_text_chunk(key, json.dumps(value))) + if not chunks: + return png_bytes + + # IHDR is always the first chunk; insert ours immediately after it. + ihdr_length = struct.unpack(">I", png_bytes[8:12])[0] + ihdr_end = 8 + 8 + ihdr_length + 4 # signature + (len+type) + data + crc + return png_bytes[:ihdr_end] + b"".join(chunks) + png_bytes[ihdr_end:] + + +# Standard chromaticities (CIE 1931 xy) for the colorspaces this node writes. +# Each tuple is (Rx, Ry, Gx, Gy, Bx, By, Wx, Wy). All share D65 white point. +_CHROMATICITIES = { + # ITU-R BT.709 / sRGB primaries + "Rec.709": (0.6400, 0.3300, 0.3000, 0.6000, 0.1500, 0.0600, 0.3127, 0.3290), + # ITU-R BT.2020 (UHDTV / wide-gamut HDR) primaries + "Rec.2020": (0.7080, 0.2920, 0.1700, 0.7970, 0.1310, 0.0460, 0.3127, 0.3290), +} + + +def _pack_chromaticities(primaries: tuple) -> bytes: + """Serialize 8 chromaticity floats into the EXR `chromaticities` payload.""" + return struct.pack("<8f", *primaries) + + +def _exr_attribute(name: str, attr_type: str, value: bytes) -> bytes: + """Serialize one EXR header attribute: name\\0 type\\0 size:int32 value.""" + return ( + name.encode("utf-8") + b"\x00" + + attr_type.encode("utf-8") + b"\x00" + + struct.pack(" bytes: + """Insert ComfyUI metadata and color-space info into an EXR header. + + Color: EXR pixels are linear by convention. The standard way to describe + their RGB→XYZ relationship is the `chromaticities` attribute. We pick the + primaries that match what the user told us their input was: + + colorspace="sRGB" → Rec. 709 / sRGB primaries (D65) + colorspace="HDR" → Rec. 2020 / BT.2100 primaries (D65) + + Pixels are always converted to linear scene light upstream (sRGB EOTF + inverse for sRGB; HLG OETF inverse for HDR), so the file content is + scene-linear in the indicated gamut. OpenEXR has no standard transfer- + function attribute (the OpenEXR TSC has discussed adding one but it + doesn't exist), so we don't invent one — `chromaticities` plus the EXR + linear-by-convention rule fully specifies the color. + + Prompt/workflow: written as plain `string` attributes using the same keys + (`prompt`, `workflow`, ...) that Comfy uses for PNG tEXt chunks, so the + same readers can pull them out symmetrically. + + Implementation note: the chunk-offset table that follows the header stores + *absolute* byte offsets into the file. Inserting N bytes into the header + means every offset must be incremented by N or the file becomes unreadable. + """ + if len(exr_bytes) < 8 or exr_bytes[:4] != b"\x76\x2f\x31\x01": + return exr_bytes + + new_blob = b"" + if prompt is not None: + new_blob += _exr_attribute("prompt", "string", json.dumps(prompt).encode("utf-8")) + if extra_pnginfo: + for key, value in extra_pnginfo.items(): + new_blob += _exr_attribute(key, "string", json.dumps(value).encode("utf-8")) + if colorspace is not None: + # Map each colorspace option to the RGB primaries the linear pixels + # are now in. "sRGB" and "linear" both produce Rec. 709 linear; "HDR" + # (HLG-encoded Rec. 2020 input) produces Rec. 2020 linear. + primaries_name = { + "sRGB": "Rec.709", + "linear": "Rec.709", + "HDR": "Rec.2020", + }.get(colorspace, "Rec.709") + new_blob += _exr_attribute( + "chromaticities", + "chromaticities", + _pack_chromaticities(_CHROMATICITIES[primaries_name]), + ) + if not new_blob: + return exr_bytes + + # Walk header attributes to find the terminating null byte, and pick up + # dataWindow + compression so we know how many chunks the offset table has. + pos = 8 # past magic (4) + version (4) + data_window = None + compression = 0 + while pos < len(exr_bytes) and exr_bytes[pos] != 0: + name_end = exr_bytes.index(b"\x00", pos) + attr_name = exr_bytes[pos:name_end].decode("latin-1", errors="replace") + type_end = exr_bytes.index(b"\x00", name_end + 1) + attr_type = exr_bytes[name_end + 1:type_end].decode("latin-1", errors="replace") + size = struct.unpack(" bytes: + """Encode a single HxWxC (or channel-less HxW grayscale) tensor to PNG or + EXR bytes in memory. Grayscale is written as single-channel PNG / Y-only EXR. + + For EXR the input is interpreted according to `colorspace` and converted + to scene-linear (EXR's convention) before writing: + + "sRGB" → input is sRGB-encoded Rec. 709; apply inverse sRGB EOTF. + "HDR" → input is HLG-encoded Rec. 2020 (BT.2100); apply inverse HLG + OETF to get scene-linear, per BT.2100 Note 5a. + "linear" → input is already scene-linear (Rec. 709 primaries); write + through unchanged. Use this for renderer/compositor output. + + For PNG, colorspace selection does not modify pixels — PNG is delivered + sRGB-encoded and there is no PNG path for wide-gamut HDR in this node. + """ + if img_tensor.ndim == 2: + img_tensor = img_tensor.unsqueeze(-1) # Some nodes emit grayscale as (H, W) with no channel dim, mask-style. + height, width, num_channels = img_tensor.shape + + spec = _FORMAT_SPECS.get((file_format, bit_depth, num_channels)) + if spec is None: + raise ValueError( + f"No {file_format}/{bit_depth} encoder for {num_channels}-channel images: " + "supported channel counts are 1 (grayscale), 3 (RGB) and 4 (RGBA)." + ) + + if spec["dtype"] == np.float32: + # EXR path: preserve full range, no clamp. + if colorspace == "sRGB": + img_tensor = srgb_to_linear(img_tensor) + elif colorspace == "HDR": + img_tensor = hlg_to_linear(img_tensor) + img_np = img_tensor.cpu().numpy().astype(np.float32) + else: + # PNG path: quantize to integer range. + scaled = (img_tensor * spec["scale"]).clamp(0, spec["scale"]) + img_np = scaled.to(torch.int32).cpu().numpy().astype(spec["dtype"]) + + # Encode directly via CodecContext. PyAV's `image2` muxer does NOT write to + # BytesIO (it expects a real file path), so we bypass the container entirely. + # For single-frame PNG/EXR the raw codec output IS the file. + codec = av.CodecContext.create(file_format, "w") + codec.width = width + codec.height = height + codec.pix_fmt = spec["stream_fmt"] + codec.time_base = Fraction(1, 1) + + frame = av.VideoFrame.from_ndarray(img_np, format=spec["frame_fmt"]) + if spec["frame_fmt"] != spec["stream_fmt"]: + frame = frame.reformat(format=spec["stream_fmt"]) + frame.pts = 0 + frame.time_base = codec.time_base + + packets = list(codec.encode(frame)) + list(codec.encode(None)) # flush with None + return b"".join(bytes(p) for p in packets) + + +# --------------------------------------------------------------------------- +# Node +# --------------------------------------------------------------------------- + +class SaveImageAdvanced(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="SaveImageAdvanced", + search_aliases=["save", "save image", "export image", "output image", "write image"], + display_name="Save Image (Advanced)", + description="Saves the input images to your ComfyUI output directory.", + category="image", + essentials_category="Basics", + inputs=[ + IO.Image.Input("images", tooltip="The images to save."), + IO.String.Input( + "filename_prefix", + default="ComfyUI", + tooltip=("The prefix for the file to save. May include formatting tokens such as %date:yyyy-MM-dd% or %Empty Latent Image.width%."), + ), + IO.DynamicCombo.Input( + "format", + options=[ + IO.DynamicCombo.Option("png", [ + IO.Combo.Input("bit_depth", options=["8-bit", "16-bit"], default="8-bit", advanced=True), + IO.Combo.Input("input_color_space", options=["sRGB"], default="sRGB", advanced=True), + ]), + IO.DynamicCombo.Option("exr", [ + IO.Combo.Input("bit_depth", options=["32-bit float"], default="32-bit float", advanced=True), + IO.Combo.Input( + "input_color_space", + options=["sRGB", "HDR", "linear"], + default="sRGB", + advanced=True, + tooltip=( + "Colorspace of the input tensor. The EXR is always written as scene-linear in the matching gamut.\n" + "sRGB — input is sRGB-encoded Rec.709; the inverse sRGB EOTF is applied.\n" + "HDR — input is HLG-encoded Rec.2020 (BT.2100); the inverse HLG OETF is applied to get scene-linear light.\n" + "linear — input is already scene-linear (Rec.709 primaries); written through unchanged. Use this for renderer/compositor output." + ), + ), + ]), + ], + tooltip="The file format in which to save the image.", + ), + ], + hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], + is_output_node=True, + outputs=[IO.Image.Output(display_name="images")] + ) + + @classmethod + def execute(cls, images, filename_prefix: str, format: dict) -> IO.NodeOutput: + file_format = format["format"] + bit_depth = format["bit_depth"] + colorspace = format.get("input_color_space", "sRGB") + + output_dir = folder_paths.get_output_directory() + full_output_folder, filename, counter, subfolder, filename_prefix = ( + folder_paths.get_save_image_path( + filename_prefix, output_dir, images[0].shape[1], images[0].shape[0] + ) + ) + + prompt = cls.hidden.prompt + extra_pnginfo = cls.hidden.extra_pnginfo + write_metadata = not args.disable_metadata + + results = [] + for batch_number, image in enumerate(images): + encoded = _encode_image(image, file_format, bit_depth, colorspace) + + if write_metadata: + if file_format == "png": + encoded = inject_png_metadata(encoded, prompt, extra_pnginfo) + elif file_format == "exr": + encoded = inject_exr_metadata(encoded, prompt, extra_pnginfo, colorspace) + + name = filename.replace("%batch_num%", str(batch_number)) + file = f"{name}_{counter:05}.{file_format}" + with open(os.path.join(full_output_folder, file), "wb") as f: + f.write(encoded) + + results.append({"filename": file, "subfolder": subfolder, "type": "output"}) + counter += 1 + + return IO.NodeOutput(images, ui={"images": results}) + + +class ImagesExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[IO.ComfyNode]]: + return [ + ImageCrop, + ImageCropV2, + BoundingBox, + RepeatImageBatch, + ImageFromBatch, + ImageAddNoise, + SaveAnimatedWEBP, + SaveAnimatedPNG, + SaveImageAdvanced, + SaveSVGNode, + ImageStitch, + ResizeAndPadImage, + GetImageSize, + ImageRotate, + ImageFlip, + ImageScaleToMaxDimension, + SplitImageToTileList, + ImageMergeTileList, + ] + + +async def comfy_entrypoint() -> ImagesExtension: + return ImagesExtension() diff --git a/ComfyUI/comfy_extras/nodes_ip2p.py b/ComfyUI/comfy_extras/nodes_ip2p.py new file mode 100644 index 0000000000000000000000000000000000000000..9c80834f07ec606779c64a801d543a8b94b8f29a --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_ip2p.py @@ -0,0 +1,63 @@ +import torch + +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io + + +class InstructPixToPixConditioning(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="InstructPixToPixConditioning", + category="model/conditioning/instructpix2pix", + inputs=[ + io.Conditioning.Input("positive"), + io.Conditioning.Input("negative"), + io.Vae.Input("vae"), + io.Image.Input("pixels"), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + io.Latent.Output(display_name="latent"), + ], + ) + + @classmethod + def execute(cls, positive, negative, pixels, vae) -> io.NodeOutput: + x = (pixels.shape[1] // 8) * 8 + y = (pixels.shape[2] // 8) * 8 + + if pixels.shape[1] != x or pixels.shape[2] != y: + x_offset = (pixels.shape[1] % 8) // 2 + y_offset = (pixels.shape[2] % 8) // 2 + pixels = pixels[:,x_offset:x + x_offset, y_offset:y + y_offset,:] + + concat_latent = vae.encode(pixels) + + out_latent = {} + out_latent["samples"] = torch.zeros_like(concat_latent) + + out = [] + for conditioning in [positive, negative]: + c = [] + for t in conditioning: + d = t[1].copy() + d["concat_latent_image"] = concat_latent + n = [t[0], d] + c.append(n) + out.append(c) + return io.NodeOutput(out[0], out[1], out_latent) + + +class InstructPix2PixExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + InstructPixToPixConditioning, + ] + + +async def comfy_entrypoint() -> InstructPix2PixExtension: + return InstructPix2PixExtension() + diff --git a/ComfyUI/comfy_extras/nodes_joyimage.py b/ComfyUI/comfy_extras/nodes_joyimage.py new file mode 100644 index 0000000000000000000000000000000000000000..539dc44b229ac7bd92af467965c8a4368a418ce3 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_joyimage.py @@ -0,0 +1,102 @@ +from typing_extensions import override + +import comfy.utils +import node_helpers +from comfy_api.latest import ComfyExtension, io + + +# fmt: off +BUCKETS_1024 = [ + (512, 1792), (512, 1856), (512, 1920), (512, 1984), (512, 2048), + (576, 1600), (576, 1664), (576, 1728), (576, 1792), + (640, 1472), (640, 1536), (640, 1600), + (704, 1344), (704, 1408), (704, 1472), + (768, 1216), (768, 1280), (768, 1344), + (832, 1152), (832, 1216), + (896, 1088), (896, 1152), + (960, 1024), (960, 1088), + (1024, 960), (1024, 1024), + (1088, 896), (1088, 960), + (1152, 832), (1152, 896), + (1216, 768), (1216, 832), + (1280, 768), + (1344, 704), (1344, 768), + (1408, 704), + (1472, 640), (1472, 704), + (1536, 640), + (1600, 576), (1600, 640), + (1664, 576), + (1728, 576), + (1792, 512), (1792, 576), + (1856, 512), + (1920, 512), + (1984, 512), + (2048, 512), +] +# fmt: on + + +def _find_best_bucket(height: int, width: int) -> tuple[int, int]: + target_ratio = height / width + return min(BUCKETS_1024, key=lambda hw: abs(hw[0] / hw[1] - target_ratio)) + + +def _resize_reference(image): + if image.shape[0] != 1: + raise ValueError("JoyImage reference inputs must contain one image each") + samples = image.movedim(-1, 1) + bucket_h, bucket_w = _find_best_bucket(samples.shape[2], samples.shape[3]) + resized = comfy.utils.common_upscale(samples, bucket_w, bucket_h, "bilinear", "center") + return resized.movedim(1, -1)[:, :, :, :3] + + +def _encode(clip, prompt, vae, images): + resized_images = [_resize_reference(image) for image in images] + conditioning = clip.encode_from_tokens_scheduled(clip.tokenize(prompt, images=resized_images)) + if vae is not None and resized_images: + ref_latents = [vae.encode(image) for image in resized_images] + conditioning = node_helpers.conditioning_set_values( + conditioning, {"reference_latents": ref_latents}, append=True, + ) + return conditioning + + +class TextEncodeJoyImageEdit(io.ComfyNode): + @classmethod + def define_schema(cls): + image_template = io.Autogrow.TemplatePrefix( + io.Image.Input("image"), + prefix="image", + min=0, + max=6, + ) + return io.Schema( + node_id="TextEncodeJoyImageEdit", + category="model/conditioning/joyimage", + inputs=[ + io.Clip.Input("clip"), + io.String.Input("prompt", multiline=True, dynamic_prompts=True), + io.Vae.Input("vae", optional=True), + io.Autogrow.Input("images", template=image_template, optional=True), + ], + outputs=[ + io.Conditioning.Output(), + ], + ) + + @classmethod + def execute(cls, clip, prompt, vae=None, images: io.Autogrow.Type = None) -> io.NodeOutput: + images = images or {} + return io.NodeOutput(_encode(clip, prompt, vae, list(images.values()))) + + +class JoyImageExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + TextEncodeJoyImageEdit, + ] + + +async def comfy_entrypoint() -> JoyImageExtension: + return JoyImageExtension() diff --git a/ComfyUI/comfy_extras/nodes_json_prompt.py b/ComfyUI/comfy_extras/nodes_json_prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..206f5aa710e21c6011220d5003cb4602a43dc72a --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_json_prompt.py @@ -0,0 +1,77 @@ +from typing_extensions import override + +from comfy_api.latest import ComfyExtension, io +from comfy_extras.color_util import normalize_palette + + +class BuildJsonPromptIdeogram(io.ComfyNode): + @classmethod + def define_schema(cls): + color_palette = io.Colors.Input( + "color_palette", + socketless=False, + tooltip="Hex color codes that steer the image's dominant colors. Up to 16 entries.", + ) + return io.Schema( + node_id="BuildJsonPromptIdeogram", + display_name="Build JSON Prompt (Ideogram)", + category="text", + description="Build a JSON prompt for the Ideogram 4 model.", + inputs=[ + io.Array.Input("element", tooltip="Prompt elements from the node Create Bounding Boxes."), + io.String.Input("high_level_description", multiline=True, default="", + tooltip="Optional description of the image in one or two sentences. Strongly recommended."), + io.String.Input("background", multiline=True, default="", + tooltip="Mandatory description of the image background or environment."), + io.DynamicCombo.Input("style", options=[ + io.DynamicCombo.Option("none", []), + io.DynamicCombo.Option("photo", [io.String.Input("photo", default="", tooltip="Camera or lens details for photographic outputs (e.g. 35mm, f/1.4, bokeh).")]), + io.DynamicCombo.Option("art_style", [io.String.Input("art_style", default="", tooltip="Art style description (e.g. flat vector illustration, bold outlines).")]), + ]), + io.String.Input("aesthetics", default="", tooltip="Mandatory aesthetic keywords (e.g. moody, cinematic, desaturated)."), + io.String.Input("lighting", default="", tooltip="Mandatory lighting description (e.g. golden hour, rim light, dramatic shadows)."), + io.String.Input("medium", default="", tooltip="Mandatory medium type (e.g. photograph, illustration, 3d_render, painting, graphic_design). When style = photo, set to photograph."), + color_palette, + ], + outputs=[io.Dict.Output(display_name="prompt")], + is_experimental=True, + ) + + @classmethod + def execute(cls, element, style, high_level_description="", background="", + aesthetics="", lighting="", medium="", color_palette=None) -> io.NodeOutput: + elements = element if isinstance(element, list) else [] + kind = style.get("style", "none") if isinstance(style, dict) else "none" + photo = style.get("photo", "") if isinstance(style, dict) else "" + art_style = style.get("art_style", "") if isinstance(style, dict) else "" + palette = normalize_palette(color_palette or []) + + caption: dict = {} + if high_level_description.strip(): + caption["high_level_description"] = high_level_description + if kind != "none": + style_desc: dict = {"aesthetics": aesthetics, "lighting": lighting} + if kind == "photo": + style_desc["photo"] = photo + style_desc["medium"] = medium + else: + style_desc["medium"] = medium + style_desc["art_style"] = art_style + if palette: + style_desc["color_palette"] = palette + caption["style_description"] = style_desc + caption["compositional_deconstruction"] = { + "background": background, + "elements": elements, + } + return io.NodeOutput(caption) + + +class JsonPromptExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [BuildJsonPromptIdeogram] + + +async def comfy_entrypoint() -> JsonPromptExtension: + return JsonPromptExtension() diff --git a/ComfyUI/comfy_extras/nodes_kandinsky5.py b/ComfyUI/comfy_extras/nodes_kandinsky5.py new file mode 100644 index 0000000000000000000000000000000000000000..96cca0386064991f79f2bb7a06021cb67f511992 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_kandinsky5.py @@ -0,0 +1,138 @@ +import nodes +import node_helpers +import torch +import comfy.model_management +import comfy.utils + +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io + + +class Kandinsky5ImageToVideo(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="Kandinsky5ImageToVideo", + category="model/conditioning/kandinsky", + inputs=[ + io.Conditioning.Input("positive"), + io.Conditioning.Input("negative"), + io.Vae.Input("vae"), + io.Int.Input("width", default=768, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("height", default=512, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("length", default=121, min=1, max=nodes.MAX_RESOLUTION, step=4), + io.Int.Input("batch_size", default=1, min=1, max=4096), + io.Image.Input("start_image", optional=True), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + io.Latent.Output(display_name="latent", tooltip="Empty video latent"), + io.Latent.Output(display_name="cond_latent", tooltip="Clean encoded start images, used to replace the noisy start of the model output latents"), + ], + ) + + @classmethod + def execute(cls, positive, negative, vae, width, height, length, batch_size, start_image=None) -> io.NodeOutput: + latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device()) + cond_latent_out = {} + if start_image is not None: + start_image = comfy.utils.common_upscale(start_image[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1) + encoded = vae.encode(start_image[:, :, :, :3]) + cond_latent_out["samples"] = encoded + + mask = torch.ones((1, 1, latent.shape[2], latent.shape[-2], latent.shape[-1]), device=start_image.device, dtype=start_image.dtype) + mask[:, :, :((start_image.shape[0] - 1) // 4) + 1] = 0.0 + + positive = node_helpers.conditioning_set_values(positive, {"time_dim_replace": encoded, "concat_mask": mask}) + negative = node_helpers.conditioning_set_values(negative, {"time_dim_replace": encoded, "concat_mask": mask}) + + out_latent = {} + out_latent["samples"] = latent + return io.NodeOutput(positive, negative, out_latent, cond_latent_out) + + +def adaptive_mean_std_normalization(source, reference, clump_mean_low=0.3, clump_mean_high=0.35, clump_std_low=0.35, clump_std_high=0.5): + source_mean = source.mean(dim=(1, 3, 4), keepdim=True) # mean over C, H, W + source_std = source.std(dim=(1, 3, 4), keepdim=True) # std over C, H, W + + reference_mean = torch.clamp(reference.mean(), source_mean - clump_mean_low, source_mean + clump_mean_high) + reference_std = torch.clamp(reference.std(), source_std - clump_std_low, source_std + clump_std_high) + + # normalization + normalized = (source - source_mean) / (source_std + 1e-8) + normalized = normalized * reference_std + reference_mean + + return normalized + + +class NormalizeVideoLatentStart(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="NormalizeVideoLatentStart", + category="model/conditioning", + description="Normalizes the initial frames of a video latent to match the mean and standard deviation of subsequent reference frames. Helps reduce differences between the starting frames and the rest of the video.", + inputs=[ + io.Latent.Input("latent"), + io.Int.Input("start_frame_count", default=4, min=1, max=nodes.MAX_RESOLUTION, step=1, tooltip="Number of latent frames to normalize, counted from the start"), + io.Int.Input("reference_frame_count", default=5, min=1, max=nodes.MAX_RESOLUTION, step=1, tooltip="Number of latent frames after the start frames to use as reference"), + ], + outputs=[ + io.Latent.Output(display_name="latent"), + ], + ) + + @classmethod + def execute(cls, latent, start_frame_count, reference_frame_count) -> io.NodeOutput: + if latent["samples"].shape[2] <= 1: + return io.NodeOutput(latent) + s = latent.copy() + samples = latent["samples"].clone() + + first_frames = samples[:, :, :start_frame_count] + reference_frames_data = samples[:, :, start_frame_count:start_frame_count+min(reference_frame_count, samples.shape[2]-1)] + normalized_first_frames = adaptive_mean_std_normalization(first_frames, reference_frames_data) + + samples[:, :, :start_frame_count] = normalized_first_frames + s["samples"] = samples + return io.NodeOutput(s) + + +class CLIPTextEncodeKandinsky5(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="CLIPTextEncodeKandinsky5", + display_name="CLIP Text Encode (Kandinsky 5)", + search_aliases=["kandinsky prompt"], + category="model/conditioning/kandinsky", + inputs=[ + io.Clip.Input("clip"), + io.String.Input("clip_l", multiline=True, dynamic_prompts=True), + io.String.Input("qwen25_7b", multiline=True, dynamic_prompts=True), + ], + outputs=[ + io.Conditioning.Output(), + ], + ) + + @classmethod + def execute(cls, clip, clip_l, qwen25_7b) -> io.NodeOutput: + tokens = clip.tokenize(clip_l) + tokens["qwen25_7b"] = clip.tokenize(qwen25_7b)["qwen25_7b"] + + return io.NodeOutput(clip.encode_from_tokens_scheduled(tokens)) + + +class Kandinsky5Extension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + Kandinsky5ImageToVideo, + NormalizeVideoLatentStart, + CLIPTextEncodeKandinsky5, + ] + +async def comfy_entrypoint() -> Kandinsky5Extension: + return Kandinsky5Extension() diff --git a/ComfyUI/comfy_extras/nodes_latent.py b/ComfyUI/comfy_extras/nodes_latent.py new file mode 100644 index 0000000000000000000000000000000000000000..1f93e34d647ac53ee499376844c8ea178bbccca3 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_latent.py @@ -0,0 +1,506 @@ +import comfy.utils +import comfy_extras.nodes_post_processing +import torch +import nodes +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io +import logging +import math + +def reshape_latent_to(target_shape, latent, repeat_batch=True): + if latent.shape[1:] != target_shape[1:]: + latent = comfy.utils.common_upscale(latent, target_shape[-1], target_shape[-2], "bilinear", "center") + if repeat_batch: + return comfy.utils.repeat_to_batch_size(latent, target_shape[0]) + else: + return latent + + +class LatentAdd(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LatentAdd", + search_aliases=["combine latents", "sum latents"], + category="model/latent/advanced", + inputs=[ + io.Latent.Input("samples1"), + io.Latent.Input("samples2"), + ], + outputs=[ + io.Latent.Output(), + ], + ) + + @classmethod + def execute(cls, samples1, samples2) -> io.NodeOutput: + samples_out = samples1.copy() + + s1 = samples1["samples"] + s2 = samples2["samples"] + + s2 = reshape_latent_to(s1.shape, s2) + samples_out["samples"] = s1 + s2 + return io.NodeOutput(samples_out) + +class LatentSubtract(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LatentSubtract", + search_aliases=["difference latent", "remove features"], + category="model/latent/advanced", + inputs=[ + io.Latent.Input("samples1"), + io.Latent.Input("samples2"), + ], + outputs=[ + io.Latent.Output(), + ], + ) + + @classmethod + def execute(cls, samples1, samples2) -> io.NodeOutput: + samples_out = samples1.copy() + + s1 = samples1["samples"] + s2 = samples2["samples"] + + s2 = reshape_latent_to(s1.shape, s2) + samples_out["samples"] = s1 - s2 + return io.NodeOutput(samples_out) + +class LatentMultiply(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LatentMultiply", + search_aliases=["scale latent", "amplify latent", "latent gain"], + category="model/latent/advanced", + inputs=[ + io.Latent.Input("samples"), + io.Float.Input("multiplier", default=1.0, min=-10.0, max=10.0, step=0.01), + ], + outputs=[ + io.Latent.Output(), + ], + ) + + @classmethod + def execute(cls, samples, multiplier) -> io.NodeOutput: + samples_out = samples.copy() + + s1 = samples["samples"] + samples_out["samples"] = s1 * multiplier + return io.NodeOutput(samples_out) + +class LatentInterpolate(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LatentInterpolate", + search_aliases=["blend latent", "mix latent", "lerp latent", "transition"], + category="model/latent/advanced", + inputs=[ + io.Latent.Input("samples1"), + io.Latent.Input("samples2"), + io.Float.Input("ratio", default=1.0, min=0.0, max=1.0, step=0.01), + ], + outputs=[ + io.Latent.Output(), + ], + ) + + @classmethod + def execute(cls, samples1, samples2, ratio) -> io.NodeOutput: + samples_out = samples1.copy() + + s1 = samples1["samples"] + s2 = samples2["samples"] + + s2 = reshape_latent_to(s1.shape, s2) + + m1 = torch.linalg.vector_norm(s1, dim=(1)) + m2 = torch.linalg.vector_norm(s2, dim=(1)) + + s1 = torch.nan_to_num(s1 / m1) + s2 = torch.nan_to_num(s2 / m2) + + t = (s1 * ratio + s2 * (1.0 - ratio)) + mt = torch.linalg.vector_norm(t, dim=(1)) + st = torch.nan_to_num(t / mt) + + samples_out["samples"] = st * (m1 * ratio + m2 * (1.0 - ratio)) + return io.NodeOutput(samples_out) + +class LatentConcat(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LatentConcat", + search_aliases=["join latents", "stitch latents"], + category="model/latent/advanced", + inputs=[ + io.Latent.Input("samples1"), + io.Latent.Input("samples2"), + io.Combo.Input("dim", options=["x", "-x", "y", "-y", "t", "-t"]), + ], + outputs=[ + io.Latent.Output(), + ], + ) + + @classmethod + def execute(cls, samples1, samples2, dim) -> io.NodeOutput: + samples_out = samples1.copy() + + s1 = samples1["samples"] + s2 = samples2["samples"] + s2 = comfy.utils.repeat_to_batch_size(s2, s1.shape[0]) + + if "-" in dim: + c = (s2, s1) + else: + c = (s1, s2) + + if "x" in dim: + dim = -1 + elif "y" in dim: + dim = -2 + elif "t" in dim: + dim = -3 + + samples_out["samples"] = torch.cat(c, dim=dim) + return io.NodeOutput(samples_out) + +class LatentCut(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LatentCut", + search_aliases=["crop latent", "slice latent", "extract region"], + category="model/latent/advanced", + inputs=[ + io.Latent.Input("samples"), + io.Combo.Input("dim", options=["x", "y", "t"]), + io.Int.Input("index", default=0, min=-nodes.MAX_RESOLUTION, max=nodes.MAX_RESOLUTION, step=1), + io.Int.Input("amount", default=1, min=1, max=nodes.MAX_RESOLUTION, step=1), + ], + outputs=[ + io.Latent.Output(), + ], + ) + + @classmethod + def execute(cls, samples, dim, index, amount) -> io.NodeOutput: + samples_out = samples.copy() + + s1 = samples["samples"] + + if "x" in dim: + dim = s1.ndim - 1 + elif "y" in dim: + dim = s1.ndim - 2 + elif "t" in dim: + dim = s1.ndim - 3 + + if index >= 0: + index = min(index, s1.shape[dim] - 1) + amount = min(s1.shape[dim] - index, amount) + else: + index = max(index, -s1.shape[dim]) + amount = min(-index, amount) + + samples_out["samples"] = torch.narrow(s1, dim, index, amount) + return io.NodeOutput(samples_out) + +class LatentCutToBatch(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LatentCutToBatch", + search_aliases=["slice to batch", "split latent", "tile latent"], + category="model/latent/advanced", + inputs=[ + io.Latent.Input("samples"), + io.Combo.Input("dim", options=["t", "x", "y"]), + io.Int.Input("slice_size", default=1, min=1, max=nodes.MAX_RESOLUTION, step=1), + ], + outputs=[ + io.Latent.Output(), + ], + ) + + @classmethod + def execute(cls, samples, dim, slice_size) -> io.NodeOutput: + samples_out = samples.copy() + + s1 = samples["samples"] + + if "x" in dim: + dim = s1.ndim - 1 + elif "y" in dim: + dim = s1.ndim - 2 + elif "t" in dim: + dim = s1.ndim - 3 + + if dim < 2: + return io.NodeOutput(samples) + + s = s1.movedim(dim, 1) + if s.shape[1] < slice_size: + slice_size = s.shape[1] + elif s.shape[1] % slice_size != 0: + s = s[:, :math.floor(s.shape[1] / slice_size) * slice_size] + new_shape = [-1, slice_size] + list(s.shape[2:]) + samples_out["samples"] = s.reshape(new_shape).movedim(1, dim) + return io.NodeOutput(samples_out) + +class LatentBatch(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LatentBatch", + search_aliases=["combine latents", "merge latents", "join latents"], + display_name="Batch Latents (DEPRECATED)", + category="model/latent/batch", + is_deprecated=True, + inputs=[ + io.Latent.Input("samples1"), + io.Latent.Input("samples2"), + ], + outputs=[ + io.Latent.Output(), + ], + ) + + @classmethod + def execute(cls, samples1, samples2) -> io.NodeOutput: + samples_out = samples1.copy() + s1 = samples1["samples"] + s2 = samples2["samples"] + + s2 = reshape_latent_to(s1.shape, s2, repeat_batch=False) + s = torch.cat((s1, s2), dim=0) + samples_out["samples"] = s + samples_out["batch_index"] = samples1.get("batch_index", [x for x in range(0, s1.shape[0])]) + samples2.get("batch_index", [x for x in range(0, s2.shape[0])]) + return io.NodeOutput(samples_out) + +class LatentBatchSeedBehavior(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LatentBatchSeedBehavior", + category="model/latent/advanced", + inputs=[ + io.Latent.Input("samples"), + io.Combo.Input("seed_behavior", options=["random", "fixed"], default="fixed"), + ], + outputs=[ + io.Latent.Output(), + ], + ) + + @classmethod + def execute(cls, samples, seed_behavior) -> io.NodeOutput: + samples_out = samples.copy() + latent = samples["samples"] + if seed_behavior == "random": + if 'batch_index' in samples_out: + samples_out.pop('batch_index') + elif seed_behavior == "fixed": + batch_number = samples_out.get("batch_index", [0])[0] + samples_out["batch_index"] = [batch_number] * latent.shape[0] + + return io.NodeOutput(samples_out) + +class LatentApplyOperation(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LatentApplyOperation", + search_aliases=["transform latent"], + category="model/latent/advanced/operations", + is_experimental=True, + inputs=[ + io.Latent.Input("samples"), + io.LatentOperation.Input("operation"), + ], + outputs=[ + io.Latent.Output(), + ], + ) + + @classmethod + def execute(cls, samples, operation) -> io.NodeOutput: + samples_out = samples.copy() + + s1 = samples["samples"] + samples_out["samples"] = operation(latent=s1) + return io.NodeOutput(samples_out) + +class LatentApplyOperationCFG(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LatentApplyOperationCFG", + category="model/latent/advanced/operations", + is_experimental=True, + inputs=[ + io.Model.Input("model"), + io.LatentOperation.Input("operation"), + ], + outputs=[ + io.Model.Output(), + ], + ) + + @classmethod + def execute(cls, model, operation) -> io.NodeOutput: + m = model.clone() + + def pre_cfg_function(args): + conds_out = args["conds_out"] + if len(conds_out) == 2: + conds_out[0] = operation(latent=(conds_out[0] - conds_out[1])) + conds_out[1] + else: + conds_out[0] = operation(latent=conds_out[0]) + return conds_out + + m.set_model_sampler_pre_cfg_function(pre_cfg_function) + return io.NodeOutput(m) + +class LatentOperationTonemapReinhard(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LatentOperationTonemapReinhard", + search_aliases=["hdr latent"], + category="model/latent/advanced/operations", + is_experimental=True, + inputs=[ + io.Float.Input("multiplier", default=1.0, min=0.0, max=100.0, step=0.01), + ], + outputs=[ + io.LatentOperation.Output(), + ], + ) + + @classmethod + def execute(cls, multiplier) -> io.NodeOutput: + def tonemap_reinhard(latent, **kwargs): + latent_vector_magnitude = (torch.linalg.vector_norm(latent, dim=(1)) + 0.0000000001)[:,None] + normalized_latent = latent / latent_vector_magnitude + + dims = list(range(1, latent_vector_magnitude.ndim)) + mean = torch.mean(latent_vector_magnitude, dim=dims, keepdim=True) + std = torch.std(latent_vector_magnitude, dim=dims, keepdim=True) + + top = (std * 5 + mean) * multiplier + + #reinhard + latent_vector_magnitude *= (1.0 / top) + new_magnitude = latent_vector_magnitude / (latent_vector_magnitude + 1.0) + new_magnitude *= top + + return normalized_latent * new_magnitude + return io.NodeOutput(tonemap_reinhard) + +class LatentOperationSharpen(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LatentOperationSharpen", + category="model/latent/advanced/operations", + is_experimental=True, + inputs=[ + io.Int.Input("sharpen_radius", default=9, min=1, max=31, step=1, advanced=True), + io.Float.Input("sigma", default=1.0, min=0.1, max=10.0, step=0.1, advanced=True), + io.Float.Input("alpha", default=0.1, min=0.0, max=5.0, step=0.01, advanced=True), + ], + outputs=[ + io.LatentOperation.Output(), + ], + ) + + @classmethod + def execute(cls, sharpen_radius, sigma, alpha) -> io.NodeOutput: + def sharpen(latent, **kwargs): + luminance = (torch.linalg.vector_norm(latent, dim=(1)) + 1e-6)[:,None] + normalized_latent = latent / luminance + channels = latent.shape[1] + + kernel_size = sharpen_radius * 2 + 1 + kernel = comfy_extras.nodes_post_processing.gaussian_kernel(kernel_size, sigma, device=luminance.device) + center = kernel_size // 2 + + kernel *= alpha * -10 + kernel[center, center] = kernel[center, center] - kernel.sum() + 1.0 + + padded_image = torch.nn.functional.pad(normalized_latent, (sharpen_radius,sharpen_radius,sharpen_radius,sharpen_radius), 'reflect') + sharpened = torch.nn.functional.conv2d(padded_image, kernel.repeat(channels, 1, 1).unsqueeze(1), padding=kernel_size // 2, groups=channels)[:,:,sharpen_radius:-sharpen_radius, sharpen_radius:-sharpen_radius] + + return luminance * sharpened + return io.NodeOutput(sharpen) + +class ReplaceVideoLatentFrames(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ReplaceVideoLatentFrames", + display_name="Replace Video Latent Frames", + category="model/latent/batch", + inputs=[ + io.Latent.Input("destination", tooltip="The destination latent where frames will be replaced."), + io.Latent.Input("source", optional=True, tooltip="The source latent providing frames to insert into the destination latent. If not provided, the destination latent is returned unchanged."), + io.Int.Input("index", default=0, min=-nodes.MAX_RESOLUTION, max=nodes.MAX_RESOLUTION, step=1, tooltip="The starting latent frame index in the destination latent where the source latent frames will be placed. Negative values count from the end."), + ], + outputs=[ + io.Latent.Output(), + ], + ) + + @classmethod + def execute(cls, destination, index, source=None) -> io.NodeOutput: + if source is None: + return io.NodeOutput(destination) + dest_frames = destination["samples"].shape[2] + source_frames = source["samples"].shape[2] + if index < 0: + index = dest_frames + index + if index > dest_frames: + logging.warning(f"ReplaceVideoLatentFrames: Index {index} is out of bounds for destination latent frames {dest_frames}.") + return io.NodeOutput(destination) + if index + source_frames > dest_frames: + logging.warning(f"ReplaceVideoLatentFrames: Source latent frames {source_frames} do not fit within destination latent frames {dest_frames} at the specified index {index}.") + return io.NodeOutput(destination) + s = source.copy() + s_source = source["samples"] + s_destination = destination["samples"].clone() + s_destination[:, :, index:index + s_source.shape[2]] = s_source + s["samples"] = s_destination + return io.NodeOutput(s) + +class LatentExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + LatentAdd, + LatentSubtract, + LatentMultiply, + LatentInterpolate, + LatentConcat, + LatentCut, + LatentCutToBatch, + LatentBatch, + LatentBatchSeedBehavior, + LatentApplyOperation, + LatentApplyOperationCFG, + LatentOperationTonemapReinhard, + LatentOperationSharpen, + ReplaceVideoLatentFrames + ] + + +async def comfy_entrypoint() -> LatentExtension: + return LatentExtension() diff --git a/ComfyUI/comfy_extras/nodes_load_3d.py b/ComfyUI/comfy_extras/nodes_load_3d.py new file mode 100644 index 0000000000000000000000000000000000000000..106b01f9dea57ca8ba5630c70764b072df1314a5 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_load_3d.py @@ -0,0 +1,400 @@ +import nodes +import folder_paths +import os +import uuid + +from typing_extensions import override +from comfy_api.latest import IO, UI, ComfyExtension, InputImpl, Types + +from pathlib import Path + + +def normalize_path(path): + return path.replace('\\', '/') + +class Load3D(IO.ComfyNode): + @classmethod + def define_schema(cls): + input_dir = os.path.join(folder_paths.get_input_directory(), "3d") + + os.makedirs(input_dir, exist_ok=True) + + input_path = Path(input_dir) + base_path = Path(folder_paths.get_input_directory()) + + files = [ + normalize_path(str(file_path.relative_to(base_path))) + for file_path in input_path.rglob("*") + if file_path.suffix.lower() in {'.gltf', '.glb', '.obj', '.fbx', '.stl', '.spz', '.splat', '.ply', '.ksplat'} + ] + return IO.Schema( + node_id="Load3D", + display_name="Load 3D & Animation", + category="3d", + essentials_category="Basics", + is_experimental=True, + inputs=[ + IO.Combo.Input("model_file", options=["none"] + sorted(files), upload=IO.UploadType.model), + IO.Load3D.Input("image"), + IO.Int.Input("width", default=1024, min=1, max=4096, step=1), + IO.Int.Input("height", default=1024, min=1, max=4096, step=1), + ], + outputs=[ + IO.Image.Output(display_name="image"), + IO.Mask.Output(display_name="mask"), + IO.String.Output(display_name="mesh_path"), + IO.Image.Output(display_name="normal"), + IO.Load3DCamera.Output(display_name="camera_info"), + IO.Video.Output(display_name="recording_video"), + IO.File3DAny.Output(display_name="model_3d"), + IO.Load3DModelInfo.Output(display_name="model_3d_info"), + ], + ) + + @classmethod + def validate_inputs(cls, model_file, **kwargs) -> bool | str: + if not model_file or model_file == "none": + return True + if not folder_paths.exists_annotated_filepath(model_file): + return f"Invalid 3D model file: {model_file}" + return True + + @classmethod + def execute(cls, model_file, image, **kwargs) -> IO.NodeOutput: + load_image_node = nodes.LoadImage() + output_image, ignore_mask = load_image_node.load_image(image=image['image']) + ignore_image, output_mask = load_image_node.load_image(image=image['mask']) + normal_image, ignore_mask2 = load_image_node.load_image(image=image['normal']) + + video = None + + if image['recording'] != "": + recording_video_path = folder_paths.get_annotated_filepath(image['recording']) + + video = InputImpl.VideoFromFile(recording_video_path) + + file_3d = None + mesh_path = "" + if model_file and model_file != "none": + file_3d = Types.File3D(folder_paths.get_annotated_filepath(model_file)) + mesh_path = model_file + model_3d_info = image.get('model_3d_info', []) + return IO.NodeOutput(output_image, output_mask, mesh_path, normal_image, image['camera_info'], video, file_3d, model_3d_info) + + process = execute # TODO: remove + + +class Preview3D(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="Preview3D", + search_aliases=["view mesh", "3d viewer"], + display_name="Preview 3D & Animation", + category="3d", + description="Preview a 3D model file without saving it to the ComfyUI output directory.", + is_experimental=True, + is_output_node=True, + inputs=[ + IO.MultiType.Input( + IO.String.Input("model_file", default="", multiline=False), + types=[ + IO.File3DGLB, + IO.File3DGLTF, + IO.File3DFBX, + IO.File3DOBJ, + IO.File3DSTL, + IO.File3DUSDZ, + IO.File3DAny, + ], + tooltip="3D model file or path string", + ), + IO.Load3DCamera.Input("camera_info", optional=True, advanced=True), + IO.Image.Input("bg_image", optional=True, advanced=True), + ], + outputs=[], + ) + + @classmethod + def execute(cls, model_file: str | Types.File3D, **kwargs) -> IO.NodeOutput: + if isinstance(model_file, Types.File3D): + filename = f"preview3d_{uuid.uuid4().hex}.{model_file.format}" + model_file.save_to(os.path.join(folder_paths.get_output_directory(), filename)) + else: + filename = model_file + camera_info = kwargs.get("camera_info", None) + bg_image = kwargs.get("bg_image", None) + return IO.NodeOutput(ui=UI.PreviewUI3D(filename, camera_info, bg_image=bg_image)) + + process = execute # TODO: remove + + +class Preview3DAdvanced(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="Preview3DAdvanced", + display_name="Preview 3D (Advanced)", + search_aliases=["preview 3d", "3d viewer", "view mesh", "frame 3d", "3d camera output"], + category="3d", + description="Preview a 3D model file without saving it to the ComfyUI output directory.", + is_experimental=True, + is_output_node=True, + inputs=[ + IO.MultiType.Input( + "model_3d", + types=[ + IO.File3DGLB, + IO.File3DGLTF, + IO.File3DFBX, + IO.File3DOBJ, + IO.File3DSTL, + IO.File3DUSDZ, + IO.File3DAny, + ], + tooltip="3D model file from an upstream 3D node.", + ), + IO.Load3DModelInfo.Input("model_3d_info", optional=True, advanced=True), + IO.Load3D.Input("viewport_state"), + IO.Load3DCamera.Input("camera_info", optional=True, advanced=True), + IO.Int.Input("width", default=1024, min=1, max=4096, step=1), + IO.Int.Input("height", default=1024, min=1, max=4096, step=1), + ], + outputs=[ + IO.File3DAny.Output(display_name="model_3d"), + IO.Load3DModelInfo.Output(display_name="model_3d_info"), + IO.Load3DCamera.Output(display_name="camera_info"), + IO.Int.Output(display_name="width"), + IO.Int.Output(display_name="height"), + ], + ) + + @classmethod + def execute(cls, model_3d: Types.File3D, viewport_state, width: int, height: int, **kwargs) -> IO.NodeOutput: + filename = f"preview3d_advanced_{uuid.uuid4().hex}.{model_3d.format}" + model_3d.save_to(os.path.join(folder_paths.get_temp_directory(), filename)) + + viewport_state = viewport_state if isinstance(viewport_state, dict) else {} + camera_info_input = kwargs.get("camera_info", None) + camera_info = camera_info_input if camera_info_input is not None else viewport_state.get('camera_info') + model_3d_info_input = kwargs.get("model_3d_info", None) + model_3d_info = model_3d_info_input if model_3d_info_input is not None else viewport_state.get('model_3d_info', []) + return IO.NodeOutput( + model_3d, + model_3d_info, + camera_info, + width, + height, + ui=UI.PreviewUI3DAdvanced(filename, camera_info, model_3d_info), + ) + + +class PreviewGaussianSplat(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="PreviewGaussianSplat", + display_name="Preview Splat", + category="3d", + description="Preview a gaussian splat 3D file without saving it to the ComfyUI output directory.", + is_experimental=True, + is_output_node=True, + search_aliases=[ + "view splat", + "view gaussian", + "view gaussian splat", + "preview gaussian", + "preview gaussian splat", + "view 3dgs", + "preview 3dgs", + "preview ply", + "preview spz", + "preview splat", + "preview ksplat", + ], + inputs=[ + IO.MultiType.Input( + "model_3d", + types=[ + IO.File3DSplatAny, + IO.File3DPLY, + IO.File3DSPLAT, + IO.File3DSPZ, + IO.File3DKSPLAT, + ], + tooltip="A gaussian splat 3D file.", + ), + IO.Load3DModelInfo.Input("model_3d_info", optional=True, advanced=True), + IO.Load3D.Input("viewport_state"), + IO.Load3DCamera.Input("camera_info", optional=True, advanced=True), + IO.Int.Input("width", default=1024, min=1, max=4096, step=1), + IO.Int.Input("height", default=1024, min=1, max=4096, step=1), + ], + outputs=[ + IO.File3DSplatAny.Output(display_name="model_3d"), + IO.Load3DModelInfo.Output(display_name="model_3d_info"), + IO.Load3DCamera.Output(display_name="camera_info"), + IO.Int.Output(display_name="width"), + IO.Int.Output(display_name="height"), + ], + ) + + @classmethod + def execute(cls, model_3d: Types.File3D, viewport_state, width: int, height: int, **kwargs) -> IO.NodeOutput: + filename = f"preview_splat_{uuid.uuid4().hex}.{model_3d.format}" + model_3d.save_to(os.path.join(folder_paths.get_temp_directory(), filename)) + + viewport_state = viewport_state if isinstance(viewport_state, dict) else {} + camera_info_input = kwargs.get("camera_info", None) + camera_info = camera_info_input if camera_info_input is not None else viewport_state.get('camera_info') + model_3d_info_input = kwargs.get("model_3d_info", None) + model_3d_info = model_3d_info_input if model_3d_info_input is not None else viewport_state.get('model_3d_info', []) + return IO.NodeOutput( + model_3d, + model_3d_info, + camera_info, + width, + height, + ui=UI.PreviewUI3DAdvanced(filename, camera_info, model_3d_info), + ) + + +class PreviewPointCloud(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="PreviewPointCloud", + display_name="Preview Point Cloud", + category="3d", + description="Preview a point cloud 3D file without saving it to the ComfyUI output directory.", + is_experimental=True, + is_output_node=True, + search_aliases=[ + "view point cloud", + "view pointcloud", + "preview point cloud", + "preview pointcloud", + "preview ply", + ], + inputs=[ + IO.MultiType.Input( + "model_3d", + types=[ + IO.File3DPointCloudAny, + IO.File3DPLY, + ], + tooltip="Point cloud file (.ply)", + ), + IO.Load3DModelInfo.Input("model_3d_info", optional=True, advanced=True), + IO.Load3D.Input("viewport_state"), + IO.Load3DCamera.Input("camera_info", optional=True, advanced=True), + IO.Int.Input("width", default=1024, min=1, max=4096, step=1), + IO.Int.Input("height", default=1024, min=1, max=4096, step=1), + ], + outputs=[ + IO.File3DPointCloudAny.Output(display_name="model_3d"), + IO.Load3DModelInfo.Output(display_name="model_3d_info"), + IO.Load3DCamera.Output(display_name="camera_info"), + IO.Int.Output(display_name="width"), + IO.Int.Output(display_name="height"), + ], + ) + + @classmethod + def execute(cls, model_3d: Types.File3D, viewport_state, width: int, height: int, **kwargs) -> IO.NodeOutput: + filename = f"preview_pointcloud_{uuid.uuid4().hex}.{model_3d.format}" + model_3d.save_to(os.path.join(folder_paths.get_temp_directory(), filename)) + + viewport_state = viewport_state if isinstance(viewport_state, dict) else {} + camera_info_input = kwargs.get("camera_info", None) + camera_info = camera_info_input if camera_info_input is not None else viewport_state.get('camera_info') + model_3d_info_input = kwargs.get("model_3d_info", None) + model_3d_info = model_3d_info_input if model_3d_info_input is not None else viewport_state.get('model_3d_info', []) + return IO.NodeOutput( + model_3d, + model_3d_info, + camera_info, + width, + height, + ui=UI.PreviewUI3DAdvanced(filename, camera_info, model_3d_info), + ) + + +MESH_EXTENSIONS = {'.gltf', '.glb', '.obj', '.fbx', '.stl'} + + +class Load3DAdvanced(IO.ComfyNode): + @classmethod + def define_schema(cls): + input_dir = os.path.join(folder_paths.get_input_directory(), "3d") + os.makedirs(input_dir, exist_ok=True) + + input_path = Path(input_dir) + base_path = Path(folder_paths.get_input_directory()) + + files = [ + normalize_path(str(file_path.relative_to(base_path))) + for file_path in input_path.rglob("*") + if file_path.suffix.lower() in MESH_EXTENSIONS + ] + return IO.Schema( + node_id="Load3DAdvanced", + display_name="Load 3D (Advanced)", + category="3d", + search_aliases=[ + "load mesh", + "load gltf", + "load glb", + "load obj", + "load fbx", + "load stl", + ], + is_experimental=True, + inputs=[ + IO.Combo.Input("model_file", options=["none"] + sorted(files), upload=IO.UploadType.model), + IO.Load3D.Input("viewport_state"), + IO.Int.Input("width", default=1024, min=1, max=4096, step=1), + IO.Int.Input("height", default=1024, min=1, max=4096, step=1), + ], + outputs=[ + IO.File3DAny.Output(display_name="model_3d"), + IO.Load3DModelInfo.Output(display_name="model_3d_info"), + IO.Load3DCamera.Output(display_name="camera_info"), + IO.Int.Output(display_name="width"), + IO.Int.Output(display_name="height"), + ], + ) + + @classmethod + def validate_inputs(cls, model_file, **kwargs) -> bool | str: + if not model_file or model_file == "none": + return True + if not folder_paths.exists_annotated_filepath(model_file): + return f"Invalid 3D model file: {model_file}" + return True + + @classmethod + def execute(cls, model_file, viewport_state, width: int, height: int, **kwargs) -> IO.NodeOutput: + file_3d = None + if model_file and model_file != "none": + file_3d = Types.File3D(folder_paths.get_annotated_filepath(model_file)) + viewport_state = viewport_state if isinstance(viewport_state, dict) else {} + model_3d_info = viewport_state.get('model_3d_info', []) + return IO.NodeOutput(file_3d, model_3d_info, viewport_state.get('camera_info'), width, height) + + +class Load3DExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[IO.ComfyNode]]: + return [ + Load3D, + Load3DAdvanced, + Preview3D, + Preview3DAdvanced, + PreviewGaussianSplat, + PreviewPointCloud, + ] + + +async def comfy_entrypoint() -> Load3DExtension: + return Load3DExtension() diff --git a/ComfyUI/comfy_extras/nodes_logic.py b/ComfyUI/comfy_extras/nodes_logic.py new file mode 100644 index 0000000000000000000000000000000000000000..13c1685f7d8ba95d9d5ab640f8249e0f895889d5 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_logic.py @@ -0,0 +1,353 @@ +from typing import TypedDict +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io +from comfy_api.latest import _io + +# sentinel for missing inputs +MISSING = object() + + +class NotNode(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ComfyNotNode", + display_name="Not", + category="utilities/logic", + description="Logical NOT operation. Returns true if the value is falsy. Uses Python's rules for truthiness.", + search_aliases=["invert", "toggle", "negate", "flip boolean"], + inputs=[ + io.AnyType.Input("value"), + ], + outputs=[ + io.Boolean.Output(), + ], + ) + + @classmethod + def execute(cls, value) -> io.NodeOutput: + return io.NodeOutput(not value) + + +class AndNode(io.ComfyNode): + @classmethod + def define_schema(cls): + template = io.Autogrow.TemplatePrefix( + input=io.AnyType.Input("value"), + prefix="value", + min=1, + ) + return io.Schema( + node_id="ComfyAndNode", + display_name="And", + category="utilities/logic", + description="Logical AND operation. Returns true if all of the values are truthy. Uses Python's rules for truthiness.", + search_aliases=["all", "every"], + inputs=[ + io.Autogrow.Input("values", template=template), + ], + outputs=[ + io.Boolean.Output(), + ], + ) + + @classmethod + def execute(cls, values: io.Autogrow.Type) -> io.NodeOutput: + return io.NodeOutput(all(values.values())) + + +class OrNode(io.ComfyNode): + @classmethod + def define_schema(cls): + template = io.Autogrow.TemplatePrefix( + input=io.AnyType.Input("value"), + prefix="value", + min=1, + ) + return io.Schema( + node_id="ComfyOrNode", + display_name="Or", + category="utilities/logic", + description="Logical OR operation. Returns true if any of the values are truthy. Uses Python's rules for truthiness.", + search_aliases=["any", "some"], + inputs=[ + io.Autogrow.Input("values", template=template), + ], + outputs=[ + io.Boolean.Output(), + ], + ) + + @classmethod + def execute(cls, values: io.Autogrow.Type) -> io.NodeOutput: + return io.NodeOutput(any(values.values())) + + +class SwitchNode(io.ComfyNode): + @classmethod + def define_schema(cls): + template = io.MatchType.Template("switch") + return io.Schema( + node_id="ComfySwitchNode", + search_aliases=["if", "then", "switch", "conditional", "branch"], + display_name="If/Else Switch", + category="utilities/logic", + is_experimental=True, + inputs=[ + io.Boolean.Input("switch"), + io.MatchType.Input("on_false", template=template, lazy=True), + io.MatchType.Input("on_true", template=template, lazy=True), + ], + outputs=[ + io.MatchType.Output(template=template, display_name="output"), + ], + ) + + @classmethod + def check_lazy_status(cls, switch, on_false=None, on_true=None): + if switch and on_true is None: + return ["on_true"] + if not switch and on_false is None: + return ["on_false"] + + @classmethod + def execute(cls, switch, on_true, on_false) -> io.NodeOutput: + return io.NodeOutput(on_true if switch else on_false) + + +class SoftSwitchNode(io.ComfyNode): + @classmethod + def define_schema(cls): + template = io.MatchType.Template("switch") + return io.Schema( + node_id="ComfySoftSwitchNode", + display_name="Soft Switch", + category="utilities/logic", + is_experimental=True, + inputs=[ + io.Boolean.Input("switch"), + io.MatchType.Input("on_false", template=template, lazy=True, optional=True), + io.MatchType.Input("on_true", template=template, lazy=True, optional=True), + ], + outputs=[ + io.MatchType.Output(template=template, display_name="output"), + ], + ) + + @classmethod + def check_lazy_status(cls, switch, on_false=MISSING, on_true=MISSING): + # We use MISSING instead of None, as None is passed for connected-but-unevaluated inputs. + # This trick allows us to ignore the value of the switch and still be able to run execute(). + + # One of the inputs may be missing, in which case we need to evaluate the other input + if on_false is MISSING: + return ["on_true"] + if on_true is MISSING: + return ["on_false"] + # Normal lazy switch operation + if switch and on_true is None: + return ["on_true"] + if not switch and on_false is None: + return ["on_false"] + + @classmethod + def validate_inputs(cls, switch, on_false=MISSING, on_true=MISSING): + # This check happens before check_lazy_status(), so we can eliminate the case where + # both inputs are missing. + if on_false is MISSING and on_true is MISSING: + return "At least one of on_false or on_true must be connected to Switch node" + return True + + @classmethod + def execute(cls, switch, on_true=MISSING, on_false=MISSING) -> io.NodeOutput: + if on_true is MISSING: + return io.NodeOutput(on_false) + if on_false is MISSING: + return io.NodeOutput(on_true) + return io.NodeOutput(on_true if switch else on_false) + + +class CustomComboNode(io.ComfyNode): + """ + Frontend node that allows user to write their own options for a combo. + This is here to make sure the node has a backend-representation to avoid some annoyances. + """ + @classmethod + def define_schema(cls): + return io.Schema( + node_id="CustomCombo", + display_name="Custom Combo", + category="utilities", + is_experimental=True, + inputs=[io.Combo.Input("choice", options=[])], + outputs=[ + io.String.Output(display_name="STRING"), + io.Int.Output(display_name="INDEX"), + ], + accept_all_inputs=True, + ) + + @classmethod + def validate_inputs(cls, choice: io.Combo.Type, index: int = 0, **kwargs) -> bool: + # NOTE: DO NOT DO THIS unless you want to skip validation entirely on the node's inputs. + # I am doing that here because the widgets (besides the combo dropdown) on this node are fully frontend defined. + # I need to skip checking that the chosen combo option is in the options list, since those are defined by the user. + return True + + @classmethod + def execute(cls, choice: io.Combo.Type, index: int = 0, **kwargs) -> io.NodeOutput: + return io.NodeOutput(choice, index) + + +class DCTestNode(io.ComfyNode): + class DCValues(TypedDict): + combo: str + string: str + integer: int + image: io.Image.Type + subcombo: dict[str] + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="DCTestNode", + display_name="DCTest", + category="utilities/logic", + is_output_node=True, + inputs=[io.DynamicCombo.Input("combo", options=[ + io.DynamicCombo.Option("option1", [io.String.Input("string")]), + io.DynamicCombo.Option("option2", [io.Int.Input("integer")]), + io.DynamicCombo.Option("option3", [io.Image.Input("image")]), + io.DynamicCombo.Option("option4", [ + io.DynamicCombo.Input("subcombo", options=[ + io.DynamicCombo.Option("opt1", [io.Float.Input("float_x"), io.Float.Input("float_y")]), + io.DynamicCombo.Option("opt2", [io.Mask.Input("mask1", optional=True)]), + ]) + ])] + )], + outputs=[io.AnyType.Output()], + ) + + @classmethod + def execute(cls, combo: DCValues) -> io.NodeOutput: + combo_val = combo["combo"] + if combo_val == "option1": + return io.NodeOutput(combo["string"]) + elif combo_val == "option2": + return io.NodeOutput(combo["integer"]) + elif combo_val == "option3": + return io.NodeOutput(combo["image"]) + elif combo_val == "option4": + return io.NodeOutput(f"{combo['subcombo']}") + else: + raise ValueError(f"Invalid combo: {combo_val}") + + +class AutogrowNamesTestNode(io.ComfyNode): + @classmethod + def define_schema(cls): + template = _io.Autogrow.TemplateNames(input=io.Float.Input("float"), names=["a", "b", "c"]) + return io.Schema( + node_id="AutogrowNamesTestNode", + display_name="AutogrowNamesTest", + category="utilities/logic", + inputs=[ + _io.Autogrow.Input("autogrow", template=template) + ], + outputs=[io.String.Output()], + ) + + @classmethod + def execute(cls, autogrow: _io.Autogrow.Type) -> io.NodeOutput: + vals = list(autogrow.values()) + combined = ",".join([str(x) for x in vals]) + return io.NodeOutput(combined) + +class AutogrowPrefixTestNode(io.ComfyNode): + @classmethod + def define_schema(cls): + template = _io.Autogrow.TemplatePrefix(input=io.Float.Input("float"), prefix="float", min=1, max=10) + return io.Schema( + node_id="AutogrowPrefixTestNode", + display_name="AutogrowPrefixTest", + category="utilities/logic", + inputs=[ + _io.Autogrow.Input("autogrow", template=template) + ], + outputs=[io.String.Output()], + ) + + @classmethod + def execute(cls, autogrow: _io.Autogrow.Type) -> io.NodeOutput: + vals = list(autogrow.values()) + combined = ",".join([str(x) for x in vals]) + return io.NodeOutput(combined) + +class ComboOutputTestNode(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ComboOptionTestNode", + display_name="ComboOptionTest", + category="utilities/logic", + inputs=[io.Combo.Input("combo", options=["option1", "option2", "option3"]), + io.Combo.Input("combo2", options=["option4", "option5", "option6"])], + outputs=[io.Combo.Output(), io.Combo.Output()], + ) + + @classmethod + def execute(cls, combo: io.Combo.Type, combo2: io.Combo.Type) -> io.NodeOutput: + return io.NodeOutput(combo, combo2) + +class ConvertStringToComboNode(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ConvertStringToComboNode", + search_aliases=["string to dropdown", "text to combo"], + display_name="Convert String to Combo", + category="utilities/logic", + inputs=[io.String.Input("string")], + outputs=[io.Combo.Output()], + ) + + @classmethod + def execute(cls, string: str) -> io.NodeOutput: + return io.NodeOutput(string) + +class InvertBooleanNode(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="InvertBooleanNode", + search_aliases=["not", "toggle", "negate", "flip boolean"], + display_name="Invert Boolean", + category="utilities/logic", + inputs=[io.Boolean.Input("boolean")], + outputs=[io.Boolean.Output()], + ) + + @classmethod + def execute(cls, boolean: bool) -> io.NodeOutput: + return io.NodeOutput(not boolean) + +class LogicExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + SwitchNode, + CustomComboNode, + NotNode, + AndNode, + OrNode, + # SoftSwitchNode, + # ConvertStringToComboNode, + # DCTestNode, + # AutogrowNamesTestNode, + # AutogrowPrefixTestNode, + # ComboOutputTestNode, + # InvertBooleanNode, + ] + +async def comfy_entrypoint() -> LogicExtension: + return LogicExtension() diff --git a/ComfyUI/comfy_extras/nodes_lora_debug.py b/ComfyUI/comfy_extras/nodes_lora_debug.py new file mode 100644 index 0000000000000000000000000000000000000000..3f68064e5f9c1102bc49b7ca56c6605549835d57 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_lora_debug.py @@ -0,0 +1,79 @@ +import folder_paths +import comfy.utils +import comfy.sd + + +class LoraLoaderBypass: + """ + Apply LoRA in bypass mode without modifying base model weights. + + Bypass mode computes: output = base_forward(x) + lora_path(x) + This is useful for training and when model weights are offloaded. + """ + + def __init__(self): + self.loaded_lora = None + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ("MODEL", {"tooltip": "The diffusion model the LoRA will be applied to."}), + "clip": ("CLIP", {"tooltip": "The CLIP model the LoRA will be applied to."}), + "lora_name": (folder_paths.get_filename_list("loras"), {"tooltip": "The name of the LoRA."}), + "strength_model": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01, "tooltip": "How strongly to modify the diffusion model. This value can be negative."}), + "strength_clip": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01, "tooltip": "How strongly to modify the CLIP model. This value can be negative."}), + } + } + + RETURN_TYPES = ("MODEL", "CLIP") + OUTPUT_TOOLTIPS = ("The modified diffusion model.", "The modified CLIP model.") + FUNCTION = "load_lora" + + CATEGORY = "model/loaders" + DESCRIPTION = "Apply LoRA in bypass mode. Unlike regular LoRA, this doesn't modify model weights - instead it injects the LoRA computation during forward pass. Useful for training scenarios." + EXPERIMENTAL = True + + def load_lora(self, model, clip, lora_name, strength_model, strength_clip): + if strength_model == 0 and strength_clip == 0: + return (model, clip) + + lora_path = folder_paths.get_full_path_or_raise("loras", lora_name) + lora = None + if self.loaded_lora is not None: + if self.loaded_lora[0] == lora_path: + lora = self.loaded_lora[1] + else: + self.loaded_lora = None + + if lora is None: + lora = comfy.utils.load_torch_file(lora_path, safe_load=True) + self.loaded_lora = (lora_path, lora) + + model_lora, clip_lora = comfy.sd.load_bypass_lora_for_models(model, clip, lora, strength_model, strength_clip) + return (model_lora, clip_lora) + + +class LoraLoaderBypassModelOnly(LoraLoaderBypass): + @classmethod + def INPUT_TYPES(s): + return {"required": { "model": ("MODEL",), + "lora_name": (folder_paths.get_filename_list("loras"), ), + "strength_model": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01}), + }} + RETURN_TYPES = ("MODEL",) + FUNCTION = "load_lora_model_only" + + def load_lora_model_only(self, model, lora_name, strength_model): + return (self.load_lora(model, None, lora_name, strength_model, 0)[0],) + + +NODE_CLASS_MAPPINGS = { + "LoraLoaderBypass": LoraLoaderBypass, + "LoraLoaderBypassModelOnly": LoraLoaderBypassModelOnly, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "LoraLoaderBypass": "Load LoRA (Bypass) (For debugging)", + "LoraLoaderBypassModelOnly": "Load LoRA (Bypass, Model Only) (for debugging)", +} diff --git a/ComfyUI/comfy_extras/nodes_lora_extract.py b/ComfyUI/comfy_extras/nodes_lora_extract.py new file mode 100644 index 0000000000000000000000000000000000000000..bcd249c29a5a0a07d886f8da93c8c3b1343b4291 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_lora_extract.py @@ -0,0 +1,145 @@ +import torch +import comfy.model_management +import comfy.utils +import folder_paths +import os +import logging +from enum import Enum +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io +from tqdm.auto import trange + +CLAMP_QUANTILE = 0.99 + +def extract_lora(diff, rank): + conv2d = (len(diff.shape) == 4) + kernel_size = None if not conv2d else diff.size()[2:4] + conv2d_3x3 = conv2d and kernel_size != (1, 1) + out_dim, in_dim = diff.size()[0:2] + rank = min(rank, in_dim, out_dim) + + if conv2d: + if conv2d_3x3: + diff = diff.flatten(start_dim=1) + else: + diff = diff.squeeze() + + + U, S, Vh = torch.linalg.svd(diff.float()) + U = U[:, :rank] + S = S[:rank] + U = U @ torch.diag(S) + Vh = Vh[:rank, :] + + dist = torch.cat([U.flatten(), Vh.flatten()]) + hi_val = torch.quantile(dist, CLAMP_QUANTILE) + low_val = -hi_val + + U = U.clamp(low_val, hi_val) + Vh = Vh.clamp(low_val, hi_val) + if conv2d: + U = U.reshape(out_dim, rank, 1, 1) + Vh = Vh.reshape(rank, in_dim, kernel_size[0], kernel_size[1]) + return (U, Vh) + +class LORAType(Enum): + STANDARD = 0 + FULL_DIFF = 1 + +LORA_TYPES = {"standard": LORAType.STANDARD, + "full_diff": LORAType.FULL_DIFF} + +def calc_lora_model(model_diff, rank, prefix_model, prefix_lora, output_sd, lora_type, bias_diff=False): + comfy.model_management.load_models_gpu([model_diff]) + sd = model_diff.model_state_dict(filter_prefix=prefix_model) + + sd_keys = list(sd.keys()) + for index in trange(len(sd_keys), unit="weight"): + k = sd_keys[index] + op_keys = sd_keys[index].rsplit('.', 1) + if len(op_keys) < 2 or op_keys[1] not in ["weight", "bias"] or (op_keys[1] == "bias" and not bias_diff): + continue + op = comfy.utils.get_attr(model_diff.model, op_keys[0]) + if hasattr(op, "comfy_cast_weights") and not getattr(op, "comfy_patched_weights", False): + weight_diff = model_diff.patch_weight_to_device(k, model_diff.load_device, return_weight=True) + else: + weight_diff = sd[k] + + if op_keys[1] == "weight": + if lora_type == LORAType.STANDARD: + if weight_diff.ndim < 2: + if bias_diff: + output_sd["{}{}.diff".format(prefix_lora, k[len(prefix_model):-7])] = weight_diff.contiguous().half().cpu() + continue + try: + out = extract_lora(weight_diff, rank) + output_sd["{}{}.lora_up.weight".format(prefix_lora, k[len(prefix_model):-7])] = out[0].contiguous().half().cpu() + output_sd["{}{}.lora_down.weight".format(prefix_lora, k[len(prefix_model):-7])] = out[1].contiguous().half().cpu() + except: + logging.warning("Could not generate lora weights for key {}, is the weight difference a zero?".format(k)) + elif lora_type == LORAType.FULL_DIFF: + output_sd["{}{}.diff".format(prefix_lora, k[len(prefix_model):-7])] = weight_diff.contiguous().half().cpu() + + elif bias_diff and op_keys[1] == "bias": + output_sd["{}{}.diff_b".format(prefix_lora, k[len(prefix_model):-5])] = weight_diff.contiguous().half().cpu() + return output_sd + +class LoraSave(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LoraSave", + search_aliases=["export lora"], + display_name="Extract and Save Lora", + category="experimental", + inputs=[ + io.String.Input("filename_prefix", default="loras/ComfyUI_extracted_lora"), + io.Int.Input("rank", default=8, min=1, max=4096, step=1, advanced=True), + io.Combo.Input("lora_type", options=tuple(LORA_TYPES.keys()), advanced=True), + io.Boolean.Input("bias_diff", default=True, advanced=True), + io.Model.Input( + "model_diff", + tooltip="The ModelSubtract output to be converted to a lora.", + optional=True, + ), + io.Clip.Input( + "text_encoder_diff", + tooltip="The CLIPSubtract output to be converted to a lora.", + optional=True, + ), + ], + is_experimental=True, + is_output_node=True, + ) + + @classmethod + def execute(cls, filename_prefix, rank, lora_type, bias_diff, model_diff=None, text_encoder_diff=None) -> io.NodeOutput: + if model_diff is None and text_encoder_diff is None: + return io.NodeOutput() + + lora_type = LORA_TYPES.get(lora_type) + full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, folder_paths.get_output_directory()) + + output_sd = {} + if model_diff is not None: + output_sd = calc_lora_model(model_diff, rank, "diffusion_model.", "diffusion_model.", output_sd, lora_type, bias_diff=bias_diff) + if text_encoder_diff is not None: + output_sd = calc_lora_model(text_encoder_diff.patcher, rank, "", "text_encoders.", output_sd, lora_type, bias_diff=bias_diff) + + output_checkpoint = f"{filename}_{counter:05}_.safetensors" + output_checkpoint = os.path.join(full_output_folder, output_checkpoint) + + comfy.utils.save_torch_file(output_sd, output_checkpoint, metadata=None) + return io.NodeOutput() + + +class LoraSaveExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + LoraSave, + ] + + +async def comfy_entrypoint() -> LoraSaveExtension: + return LoraSaveExtension() diff --git a/ComfyUI/comfy_extras/nodes_lotus.py b/ComfyUI/comfy_extras/nodes_lotus.py new file mode 100644 index 0000000000000000000000000000000000000000..9fe4c5c7b74974a22151d8d1480d385f8d06a033 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_lotus.py @@ -0,0 +1,39 @@ +from typing_extensions import override + +import torch +import comfy.model_management as mm +from comfy_api.latest import ComfyExtension, io + + +class LotusConditioning(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LotusConditioning", + category="model/conditioning/lotus", + inputs=[], + outputs=[io.Conditioning.Output(display_name="conditioning")], + ) + + @classmethod + def execute(cls) -> io.NodeOutput: + device = mm.get_torch_device() + #lotus uses a frozen encoder and null conditioning, i'm just inlining the results of that operation since it doesn't change + #and getting parity with the reference implementation would otherwise require inference and 800mb of tensors + prompt_embeds = torch.tensor([[[-0.3134765625, -0.447509765625, -0.00823974609375, -0.22802734375, 0.1785888671875, -0.2342529296875, -0.2188720703125, -0.0089111328125, -0.31396484375, 0.196533203125, -0.055877685546875, -0.3828125, -0.0965576171875, 0.0073394775390625, -0.284423828125, 0.07470703125, -0.086181640625, -0.211181640625, 0.0599365234375, 0.10693359375, 0.0007929801940917969, -0.78076171875, -0.382568359375, -0.1851806640625, -0.140625, -0.0936279296875, -0.1229248046875, -0.152099609375, -0.203857421875, -0.2349853515625, -0.2437744140625, -0.10858154296875, -0.08990478515625, 0.08892822265625, -0.2391357421875, -0.1611328125, -0.427978515625, -0.1336669921875, -0.27685546875, -0.1781005859375, -0.3857421875, 0.251953125, -0.055999755859375, -0.0712890625, -0.00130462646484375, 0.033477783203125, -0.26416015625, 0.07171630859375, -0.0090789794921875, -0.2025146484375, -0.2763671875, -0.09869384765625, -0.45751953125, -0.23095703125, 0.004528045654296875, -0.369140625, -0.366943359375, -0.205322265625, -0.1505126953125, -0.45166015625, -0.2059326171875, 0.0168609619140625, -0.305419921875, -0.150634765625, 0.02685546875, -0.609375, -0.019012451171875, 0.050445556640625, -0.0084381103515625, -0.31005859375, -0.184326171875, -0.15185546875, 0.06732177734375, 0.150390625, -0.10919189453125, -0.08837890625, -0.50537109375, -0.389892578125, -0.0294342041015625, -0.10491943359375, -0.187255859375, -0.43212890625, -0.328125, -1.060546875, 0.011871337890625, 0.04730224609375, -0.09521484375, -0.07452392578125, -0.29296875, -0.109130859375, -0.250244140625, -0.3828125, -0.171875, -0.03399658203125, -0.15478515625, -0.1861572265625, -0.2398681640625, 0.1053466796875, -0.22314453125, -0.1932373046875, -0.18798828125, -0.430419921875, -0.05364990234375, -0.474609375, -0.261474609375, -0.1077880859375, -0.439208984375, 0.08966064453125, -0.185302734375, -0.338134765625, -0.297119140625, -0.298583984375, -0.175537109375, -0.373291015625, -0.1397705078125, -0.260498046875, -0.383544921875, -0.09979248046875, -0.319580078125, -0.06884765625, -0.4365234375, -0.183837890625, -0.393310546875, -0.002277374267578125, 0.11236572265625, -0.260498046875, -0.2242431640625, -0.19384765625, -0.51123046875, 0.03216552734375, -0.048004150390625, -0.279052734375, -0.2978515625, -0.255615234375, 0.115478515625, -4.08984375, -0.1668701171875, -0.278076171875, -0.5712890625, -0.1385498046875, -0.244384765625, -0.41455078125, -0.244140625, -0.0677490234375, -0.141357421875, -0.11590576171875, -0.1439208984375, -0.0185394287109375, -2.490234375, -0.1549072265625, -0.2305908203125, -0.3828125, -0.1173095703125, -0.08258056640625, -0.1719970703125, -0.325439453125, -0.292724609375, -0.08154296875, -0.412353515625, -0.3115234375, -0.00832366943359375, 0.00489044189453125, -0.2236328125, -0.151123046875, -0.457275390625, -0.135009765625, -0.163330078125, -0.0819091796875, 0.06689453125, 0.0209197998046875, -0.11907958984375, -0.10369873046875, -0.2998046875, -0.478759765625, -0.07940673828125, -0.01517486572265625, -0.3017578125, -0.343994140625, -0.258544921875, -0.44775390625, -0.392822265625, -0.0255584716796875, -0.2998046875, 0.10833740234375, -0.271728515625, -0.36181640625, -0.255859375, -0.2056884765625, -0.055450439453125, 0.060516357421875, -0.45751953125, -0.2322998046875, -0.1737060546875, -0.40576171875, -0.2286376953125, -0.053070068359375, -0.0283660888671875, -0.1898193359375, -4.291534423828125e-05, -0.6591796875, -0.1717529296875, -0.479736328125, -0.1400146484375, -0.40771484375, 0.154296875, 0.003101348876953125, 0.00661468505859375, -0.2073974609375, -0.493408203125, 2.171875, -0.45361328125, -0.283935546875, -0.302001953125, -0.25146484375, -0.207275390625, -0.1524658203125, -0.72998046875, -0.08203125, 0.053192138671875, -0.2685546875, 0.1834716796875, -0.270263671875, -0.091552734375, -0.08319091796875, -0.1297607421875, -0.453857421875, 0.0687255859375, 0.0268096923828125, -0.16552734375, -0.4208984375, -0.1552734375, -0.057373046875, -0.300537109375, -0.04541015625, -0.486083984375, -0.2205810546875, -0.39013671875, 0.007488250732421875, -0.005329132080078125, -0.09759521484375, -0.1448974609375, -0.21923828125, -0.429443359375, -0.40087890625, -0.19384765625, -0.064453125, -0.0306243896484375, -0.045806884765625, -0.056793212890625, 0.119384765625, -0.2073974609375, -0.356201171875, -0.168212890625, -0.291748046875, -0.289794921875, -0.205322265625, -0.419677734375, -0.478271484375, -0.2037353515625, -0.368408203125, -0.186279296875, -0.427734375, -0.1756591796875, 0.07501220703125, -0.2457275390625, -0.03692626953125, 0.003997802734375, -5.7578125, -0.01052093505859375, -0.2305908203125, -0.2252197265625, -0.197509765625, -0.1566162109375, -0.1668701171875, -0.383056640625, -0.05413818359375, 0.12188720703125, -0.369873046875, -0.0184478759765625, -0.150146484375, -0.51123046875, -0.45947265625, -0.1561279296875, 0.060455322265625, 0.043487548828125, -0.1370849609375, -0.069091796875, -0.285888671875, -0.44482421875, -0.2374267578125, -0.2191162109375, -0.434814453125, -0.0360107421875, 0.1298828125, 0.0217742919921875, -0.51220703125, -0.13525390625, -0.09381103515625, -0.276611328125, -0.171875, -0.17138671875, -0.4443359375, -0.2178955078125, -0.269775390625, -0.38623046875, -0.31591796875, -0.42333984375, -0.280029296875, -0.255615234375, -0.17041015625, 0.06268310546875, -0.1878662109375, -0.00677490234375, -0.23583984375, -0.08795166015625, -0.2232666015625, -0.1719970703125, -0.484130859375, -0.328857421875, 0.04669189453125, -0.0419921875, -0.11114501953125, 0.02313232421875, -0.0033130645751953125, -0.6005859375, 0.09051513671875, -0.1884765625, -0.262939453125, -0.375732421875, -0.525390625, -0.1170654296875, -0.3779296875, -0.242919921875, -0.419921875, 0.0665283203125, -0.343017578125, 0.06658935546875, -0.346435546875, -0.1363525390625, -0.2000732421875, -0.3837890625, 0.028167724609375, 0.043853759765625, -0.0171051025390625, -0.477294921875, -0.107421875, -0.129150390625, -0.319580078125, -0.32177734375, -0.4951171875, -0.010589599609375, -0.1778564453125, -0.40234375, -0.0810546875, 0.03314208984375, -0.13720703125, -0.31591796875, -0.048248291015625, -0.274658203125, -0.0689697265625, -0.027130126953125, -0.0953369140625, 0.146728515625, -0.38671875, -0.025390625, -0.42333984375, -0.41748046875, -0.379638671875, -0.1978759765625, -0.533203125, -0.33544921875, 0.0694580078125, -0.322998046875, -0.1876220703125, 0.0094451904296875, 0.1839599609375, -0.254150390625, -0.30078125, -0.09228515625, -0.0885009765625, 0.12371826171875, 0.1500244140625, -0.12152099609375, -0.29833984375, 0.03924560546875, -0.1470947265625, -0.1610107421875, -0.2049560546875, -0.01708984375, -0.2470703125, -0.1522216796875, -0.25830078125, 0.10870361328125, -0.302490234375, -0.2376708984375, -0.360107421875, -0.443359375, -0.0784912109375, -0.63623046875, -0.0980224609375, -0.332275390625, -0.1749267578125, -0.30859375, -0.1968994140625, -0.250244140625, -0.447021484375, -0.18408203125, -0.006908416748046875, -0.2044677734375, -0.2548828125, -0.369140625, -0.11328125, -0.1103515625, -0.27783203125, -0.325439453125, 0.01381683349609375, 0.036773681640625, -0.1458740234375, -0.34619140625, -0.232177734375, -0.0562744140625, -0.4482421875, -0.21875, -0.0855712890625, -0.276123046875, -0.1544189453125, -0.223388671875, -0.259521484375, 0.0865478515625, -0.0038013458251953125, -0.340087890625, -0.076171875, -0.25341796875, -0.0007548332214355469, -0.060455322265625, -0.352294921875, 0.035736083984375, -0.2181396484375, -0.2318115234375, -0.1707763671875, 0.018646240234375, 0.093505859375, -0.197021484375, 0.033477783203125, -0.035247802734375, 0.0440673828125, -0.2056884765625, -0.040924072265625, -0.05865478515625, 0.056884765625, -0.08807373046875, -0.10845947265625, 0.09564208984375, -0.10888671875, -0.332275390625, -0.1119384765625, -0.115478515625, 13.0234375, 0.0030040740966796875, -0.53662109375, -0.1856689453125, -0.068115234375, -0.143798828125, -0.177978515625, -0.32666015625, -0.353515625, -0.1563720703125, -0.3203125, 0.0085906982421875, -0.1043701171875, -0.365478515625, -0.303466796875, -0.34326171875, -0.410888671875, -0.03790283203125, -0.11419677734375, -0.2939453125, 0.074462890625, -0.21826171875, 0.0242767333984375, -0.226318359375, -0.353515625, -0.177734375, -0.169189453125, -0.2423095703125, -0.12115478515625, -0.07843017578125, -0.341064453125, -0.2117919921875, -0.505859375, -0.544921875, -0.3935546875, -0.10772705078125, -0.2054443359375, -0.136474609375, -0.1796875, -0.396240234375, -0.1971435546875, -0.68408203125, -0.032684326171875, -0.03863525390625, -0.0709228515625, -0.1005859375, -0.156005859375, -0.3837890625, -0.319580078125, 0.11102294921875, -0.394287109375, 0.0799560546875, -0.50341796875, -0.1572265625, 0.004131317138671875, -0.12286376953125, -0.2347412109375, -0.29150390625, -0.10321044921875, -0.286376953125, 0.018798828125, -0.152099609375, -0.321044921875, 0.0191650390625, -0.11376953125, -0.54736328125, 0.15869140625, -0.257568359375, -0.2490234375, -0.3115234375, -0.09765625, -0.350830078125, -0.36376953125, -0.0771484375, -0.2298583984375, -0.30615234375, -0.052154541015625, -0.12091064453125, -0.40283203125, -0.1649169921875, 0.0206451416015625, -0.312744140625, -0.10308837890625, -0.50341796875, -0.1754150390625, -0.2003173828125, -0.173583984375, -0.204833984375, -0.1876220703125, -0.12176513671875, -0.06201171875, -0.03485107421875, -0.20068359375, -0.21484375, -0.246337890625, -0.006587982177734375, -0.09674072265625, -0.4658203125, -0.3994140625, -0.2210693359375, -0.09588623046875, -0.126220703125, -0.09222412109375, -0.145751953125, -0.217529296875, -0.289306640625, -0.28271484375, -0.1787109375, -0.169189453125, -0.359375, -0.21826171875, -0.043792724609375, -0.205322265625, -0.2900390625, -0.055419921875, -0.1490478515625, -0.340576171875, -0.045928955078125, -0.30517578125, -0.51123046875, -0.1046142578125, -0.349853515625, -0.10882568359375, -0.16748046875, -0.267333984375, -0.122314453125, -0.0985107421875, -0.3076171875, -0.1766357421875, -0.251708984375, 0.1964111328125, -0.2220458984375, -0.2349853515625, -0.035980224609375, -0.1749267578125, -0.237060546875, -0.480224609375, -0.240234375, -0.09539794921875, -0.2481689453125, -0.389404296875, -0.1748046875, -0.370849609375, -0.010650634765625, -0.147705078125, -0.0035457611083984375, -0.32568359375, -0.29931640625, -0.1395263671875, -0.28173828125, -0.09820556640625, -0.0176239013671875, -0.05926513671875, -0.0755615234375, -0.1746826171875, -0.283203125, -0.1617431640625, -0.4404296875, 0.046234130859375, -0.183837890625, -0.052032470703125, -0.24658203125, -0.11224365234375, -0.100830078125, -0.162841796875, -0.29736328125, -0.396484375, 0.11798095703125, -0.006496429443359375, -0.32568359375, -0.347900390625, -0.04595947265625, -0.09637451171875, -0.344970703125, -0.01166534423828125, -0.346435546875, -0.2861328125, -0.1845703125, -0.276611328125, -0.01312255859375, -0.395263671875, -0.50927734375, -0.1114501953125, -0.1861572265625, -0.2158203125, -0.1812744140625, 0.055419921875, -0.294189453125, 0.06500244140625, -0.1444091796875, -0.06365966796875, -0.18408203125, -0.0091705322265625, -0.1640625, -0.1856689453125, 0.090087890625, 0.024566650390625, -0.0195159912109375, -0.5546875, -0.301025390625, -0.438232421875, -0.072021484375, 0.030517578125, -0.1490478515625, 0.04888916015625, -0.23681640625, -0.1553955078125, -0.018096923828125, -0.229736328125, -0.2919921875, -0.355712890625, -0.285400390625, -0.1756591796875, -0.08355712890625, -0.416259765625, 0.022674560546875, -0.417236328125, 0.410400390625, -0.249755859375, 0.015625, -0.033599853515625, -0.040313720703125, -0.51708984375, -0.0518798828125, -0.08843994140625, -0.2022705078125, -0.3740234375, -0.285888671875, -0.176025390625, -0.292724609375, -0.369140625, -0.08367919921875, -0.356689453125, -0.38623046875, 0.06549072265625, 0.1669921875, -0.2099609375, -0.007434844970703125, 0.12890625, -0.0040740966796875, -0.2174072265625, -0.025115966796875, -0.2364501953125, -0.1695556640625, -0.0469970703125, -0.03924560546875, -0.36181640625, -0.047515869140625, -0.3154296875, -0.275634765625, -0.25634765625, -0.061920166015625, -0.12164306640625, -0.47314453125, -0.10784912109375, -0.74755859375, -0.13232421875, -0.32421875, -0.04998779296875, -0.286376953125, 0.10345458984375, -0.1710205078125, -0.388916015625, 0.12744140625, -0.3359375, -0.302490234375, -0.238525390625, -0.1455078125, -0.15869140625, -0.2427978515625, -0.0355224609375, -0.11944580078125, -0.31298828125, 0.11456298828125, -0.287841796875, -0.5439453125, -0.3076171875, -0.08642578125, -0.2408447265625, -0.283447265625, -0.428466796875, -0.085693359375, -0.1683349609375, 0.255126953125, 0.07635498046875, -0.38623046875, -0.2025146484375, -0.1331787109375, -0.10821533203125, -0.49951171875, 0.09130859375, -0.19677734375, -0.01904296875, -0.151123046875, -0.344482421875, -0.316650390625, -0.03900146484375, 0.1397705078125, 0.1334228515625, -0.037200927734375, -0.01861572265625, -0.1351318359375, -0.07037353515625, -0.380615234375, -0.34033203125, -0.06903076171875, 0.219970703125, 0.0132598876953125, -0.15869140625, -0.6376953125, 0.158935546875, -0.5283203125, -0.2320556640625, -0.185791015625, -0.2132568359375, -0.436767578125, -0.430908203125, -0.1763916015625, -0.0007672309875488281, -0.424072265625, -0.06719970703125, -0.347900390625, -0.14453125, -0.3056640625, -0.36474609375, -0.35986328125, -0.46240234375, -0.446044921875, -0.1905517578125, -0.1114501953125, -0.42919921875, -0.0643310546875, -0.3662109375, -0.4296875, -0.10968017578125, -0.2998046875, -0.1756591796875, -0.4052734375, -0.0841064453125, -0.252197265625, -0.047393798828125, 0.00434112548828125, -0.10040283203125, -0.271484375, -0.185302734375, -0.1910400390625, 0.10260009765625, 0.01393890380859375, -0.03350830078125, -0.33935546875, -0.329345703125, 0.0574951171875, -0.18896484375, -0.17724609375, -0.42919921875, -0.26708984375, -0.4189453125, -0.149169921875, -0.265625, -0.198974609375, -0.1722412109375, 0.1563720703125, -0.20947265625, -0.267822265625, -0.06353759765625, -0.365478515625, -0.340087890625, -0.3095703125, -0.320068359375, -0.0880126953125, -0.353759765625, -0.0005812644958496094, -0.1617431640625, -0.1866455078125, -0.201416015625, -0.181396484375, -0.2349853515625, -0.384765625, -0.5244140625, 0.01227569580078125, -0.21337890625, -0.30810546875, -0.17578125, -0.3037109375, -0.52978515625, -0.1561279296875, -0.296142578125, 0.057342529296875, -0.369384765625, -0.107666015625, -0.338623046875, -0.2060546875, -0.0213775634765625, -0.394775390625, -0.219482421875, -0.125732421875, -0.03997802734375, -0.42431640625, -0.134521484375, -0.2418212890625, -0.10504150390625, 0.1552734375, 0.1126708984375, -0.1427001953125, -0.133544921875, -0.111083984375, -0.375732421875, -0.2783203125, -0.036834716796875, -0.11053466796875, 0.2471923828125, -0.2529296875, -0.56494140625, -0.374755859375, -0.326416015625, 0.2137451171875, -0.09454345703125, -0.337158203125, -0.3359375, -0.34375, -0.0999755859375, -0.388671875, 0.0103302001953125, 0.14990234375, -0.2041015625, -0.39501953125, -0.39013671875, -0.1258544921875, 0.1453857421875, -0.250732421875, -0.06732177734375, -0.10638427734375, -0.032379150390625, -0.35888671875, -0.098876953125, -0.172607421875, 0.05126953125, -0.1956787109375, -0.183837890625, -0.37060546875, 0.1556396484375, -0.34375, -0.28662109375, -0.06982421875, -0.302490234375, -0.281005859375, -0.1640625, -0.5302734375, -0.1368408203125, -0.1268310546875, -0.35302734375, -0.1473388671875, -0.45556640625, -0.35986328125, -0.273681640625, -0.2249755859375, -0.1893310546875, 0.09356689453125, -0.248291015625, -0.197998046875, -0.3525390625, -0.30126953125, -0.228271484375, -0.2421875, -0.0906982421875, 0.227783203125, -0.296875, -0.009796142578125, -0.2939453125, -0.1021728515625, -0.215576171875, -0.267822265625, -0.052642822265625, 0.203369140625, -0.1417236328125, 0.18505859375, 0.12347412109375, -0.0972900390625, -0.54052734375, -0.430419921875, -0.0906982421875, -0.5419921875, -0.22900390625, -0.0625, -0.12152099609375, -0.495849609375, -0.206787109375, -0.025848388671875, 0.039031982421875, -0.453857421875, -0.318359375, -0.426025390625, -0.3701171875, -0.2169189453125, 0.0845947265625, -0.045654296875, 0.11090087890625, 0.0012454986572265625, 0.2066650390625, -0.046356201171875, -0.2337646484375, -0.295654296875, 0.057891845703125, -0.1639404296875, -0.0535888671875, -0.2607421875, -0.1488037109375, -0.16015625, -0.54345703125, -0.2305908203125, -0.55029296875, -0.178955078125, -0.222412109375, -0.0711669921875, -0.12298583984375, -0.119140625, -0.253662109375, -0.33984375, -0.11322021484375, -0.10723876953125, -0.205078125, -0.360595703125, 0.085205078125, -0.252197265625, -0.365966796875, -0.26953125, 0.2000732421875, -0.50634765625, 0.05706787109375, -0.3115234375, 0.0242919921875, -0.1689453125, -0.2401123046875, -0.3759765625, -0.2125244140625, 0.076416015625, -0.489013671875, -0.11749267578125, -0.55908203125, -0.313232421875, -0.572265625, -0.1387939453125, -0.037078857421875, -0.385498046875, 0.0323486328125, -0.39404296875, -0.05072021484375, -0.10430908203125, -0.10919189453125, -0.28759765625, -0.37451171875, -0.016937255859375, -0.2200927734375, -0.296875, -0.0286712646484375, -0.213134765625, 0.052001953125, -0.052337646484375, -0.253662109375, 0.07269287109375, -0.2498779296875, -0.150146484375, -0.09930419921875, -0.343505859375, 0.254150390625, -0.032440185546875, -0.296142578125], [1.4111328125, 0.00757598876953125, -0.428955078125, 0.089599609375, 0.0227813720703125, -0.0350341796875, -1.0986328125, 0.194091796875, 2.115234375, -0.75439453125, 0.269287109375, -0.73486328125, -1.1025390625, -0.050262451171875, -0.5830078125, 0.0268707275390625, -0.603515625, -0.6025390625, -1.1689453125, 0.25048828125, -0.4189453125, -0.5517578125, -0.30322265625, 0.7724609375, 0.931640625, -0.1422119140625, 2.27734375, -0.56591796875, 1.013671875, -0.9638671875, -0.66796875, -0.8125, 1.3740234375, -1.060546875, -1.029296875, -1.6796875, 0.62890625, 0.49365234375, 0.671875, 0.99755859375, -1.0185546875, -0.047027587890625, -0.374267578125, 0.2354736328125, 1.4970703125, -1.5673828125, 0.448974609375, 0.2078857421875, -1.060546875, -0.171875, -0.6201171875, -0.1607666015625, 0.7548828125, -0.58935546875, -0.2052001953125, 0.060791015625, 0.200439453125, 3.154296875, -3.87890625, 2.03515625, 1.126953125, 0.1640625, -1.8447265625, 0.002620697021484375, 0.7998046875, -0.337158203125, 0.47216796875, -0.5849609375, 0.9970703125, 0.3935546875, 1.22265625, -1.5048828125, -0.65673828125, 1.1474609375, -1.73046875, -1.8701171875, 1.529296875, -0.6787109375, -1.4453125, 1.556640625, -0.327392578125, 2.986328125, -0.146240234375, -2.83984375, 0.303466796875, -0.71728515625, -0.09698486328125, -0.2423095703125, 0.6767578125, -2.197265625, -0.86279296875, -0.53857421875, -1.2236328125, 1.669921875, -1.1689453125, -0.291259765625, -0.54736328125, -0.036346435546875, 1.041015625, -1.7265625, -0.6064453125, -0.1634521484375, 0.2381591796875, 0.65087890625, -1.169921875, 1.9208984375, 0.5634765625, 0.37841796875, 0.798828125, -1.021484375, -0.4091796875, 2.275390625, -0.302734375, -1.7783203125, 1.0458984375, 1.478515625, 0.708984375, -1.541015625, -0.0006041526794433594, 1.1884765625, 2.041015625, 0.560546875, -0.1131591796875, 1.0341796875, 0.06121826171875, 2.6796875, -0.53369140625, -1.2490234375, -0.7333984375, -1.017578125, -1.0078125, 1.3212890625, -0.47607421875, -1.4189453125, 0.54052734375, -0.796875, -0.73095703125, -1.412109375, -0.94873046875, -2.2734375, -1.1220703125, -1.3837890625, -0.5087890625, -1.0380859375, -0.93603515625, -0.58349609375, -1.0703125, -1.10546875, -2.60546875, 0.062225341796875, 0.38232421875, -0.411376953125, -0.369140625, -0.9833984375, -0.7294921875, -0.181396484375, -0.47216796875, -0.56884765625, -0.11041259765625, -2.673828125, 0.27783203125, -0.857421875, 0.9296875, 1.9580078125, 0.1385498046875, -1.91796875, -1.529296875, 0.53857421875, 0.509765625, -0.90380859375, -0.0947265625, -2.083984375, 0.9228515625, -0.28564453125, -0.80859375, -0.093505859375, -0.6015625, -1.255859375, 0.6533203125, 0.327880859375, -0.07598876953125, -0.22705078125, -0.30078125, -0.5185546875, -1.6044921875, 1.5927734375, 1.416015625, -0.91796875, -0.276611328125, -0.75830078125, -1.1689453125, -1.7421875, 1.0546875, -0.26513671875, -0.03314208984375, 0.278076171875, -1.337890625, 0.055023193359375, 0.10546875, -1.064453125, 1.048828125, -1.4052734375, -1.1240234375, -0.51416015625, -1.05859375, -1.7265625, -1.1328125, 0.43310546875, -2.576171875, -2.140625, -0.79345703125, 0.50146484375, 1.96484375, 0.98583984375, 0.337646484375, -0.77978515625, 0.85498046875, -0.65185546875, -0.484375, 2.708984375, 0.55810546875, -0.147216796875, -0.5537109375, -0.75439453125, -1.736328125, 1.1259765625, -1.095703125, -0.2587890625, 2.978515625, 0.335205078125, 0.357666015625, -0.09356689453125, 0.295654296875, -0.23779296875, 1.5751953125, 0.10400390625, 1.7001953125, -0.72900390625, -1.466796875, -0.2012939453125, 0.634765625, -0.1556396484375, -2.01171875, 0.32666015625, 0.047454833984375, -0.1671142578125, -0.78369140625, -0.994140625, 0.7802734375, -0.1429443359375, -0.115234375, 0.53271484375, -0.96142578125, -0.064208984375, 1.396484375, 1.654296875, -1.6015625, -0.77392578125, 0.276123046875, -0.42236328125, 0.8642578125, 0.533203125, 0.397216796875, -1.21484375, 0.392578125, -0.501953125, -0.231689453125, 1.474609375, 1.6669921875, 1.8662109375, -1.2998046875, 0.223876953125, -0.51318359375, -0.437744140625, -1.16796875, -0.7724609375, 1.6826171875, 0.62255859375, 2.189453125, -0.599609375, -0.65576171875, -1.1005859375, -0.45263671875, -0.292236328125, 2.58203125, -1.3779296875, 0.23486328125, -1.708984375, -1.4111328125, -0.5078125, -0.8525390625, -0.90771484375, 0.861328125, -2.22265625, -1.380859375, 0.7275390625, 0.85595703125, -0.77978515625, 2.044921875, -0.430908203125, 0.78857421875, -1.21484375, -0.09130859375, 0.5146484375, -1.92578125, -0.1396484375, 0.289306640625, 0.60498046875, 0.93896484375, -0.09295654296875, -0.45751953125, -0.986328125, -0.66259765625, 1.48046875, 0.274169921875, -0.267333984375, -1.3017578125, -1.3623046875, -1.982421875, -0.86083984375, -0.41259765625, -0.2939453125, -1.91015625, 1.6826171875, 0.437255859375, 1.0029296875, 0.376220703125, -0.010467529296875, -0.82861328125, -0.513671875, -3.134765625, 1.0205078125, -1.26171875, -1.009765625, 1.0869140625, -0.95703125, 0.0103759765625, 1.642578125, 0.78564453125, 1.029296875, 0.496826171875, 1.2880859375, 0.5234375, 0.05322265625, -0.206787109375, -0.79443359375, -1.1669921875, 0.049530029296875, -0.27978515625, 0.0237884521484375, -0.74169921875, -1.068359375, 0.86083984375, 1.1787109375, 0.91064453125, -0.453857421875, -1.822265625, -0.9228515625, -0.50048828125, 0.359130859375, 0.802734375, -1.3564453125, -0.322509765625, -1.1123046875, -1.0390625, -0.52685546875, -1.291015625, -0.343017578125, -1.2109375, -0.19091796875, 2.146484375, -0.04315185546875, -0.3701171875, -2.044921875, -0.429931640625, -0.56103515625, -0.166015625, -0.4658203125, -2.29296875, -1.078125, -1.0927734375, -0.1033935546875, -0.56103515625, -0.05743408203125, -1.986328125, -0.513671875, 0.70361328125, -2.484375, -1.3037109375, -1.6650390625, 0.4814453125, -0.84912109375, -2.697265625, -0.197998046875, 0.0869140625, -0.172607421875, -1.326171875, -1.197265625, 1.23828125, -0.38720703125, -0.075927734375, 0.02569580078125, -1.2119140625, 0.09027099609375, -2.12890625, -1.640625, -0.1524658203125, 0.2373046875, 1.37109375, 2.248046875, 1.4619140625, 0.3134765625, 0.50244140625, -0.1383056640625, -1.2705078125, 0.7353515625, 0.65771484375, -0.431396484375, -1.341796875, 0.10089111328125, 0.208984375, -0.0099945068359375, 0.83203125, 1.314453125, -0.422607421875, -1.58984375, -0.6044921875, 0.23681640625, -1.60546875, -0.61083984375, -1.5615234375, 1.62890625, -0.6728515625, -0.68212890625, -0.5224609375, -0.9150390625, -0.468994140625, 0.268310546875, 0.287353515625, -0.025543212890625, 0.443603515625, 1.62109375, -1.08984375, -0.5556640625, 1.03515625, -0.31298828125, -0.041778564453125, 0.260986328125, 0.34716796875, -2.326171875, 0.228271484375, -0.85107421875, -2.255859375, 0.3486328125, -0.25830078125, -0.3671875, -0.796875, -1.115234375, 1.8369140625, -0.19775390625, -1.236328125, -0.0447998046875, 0.69921875, 1.37890625, 1.11328125, 0.0928955078125, 0.6318359375, -0.62353515625, 0.55859375, -0.286865234375, 1.5361328125, -0.391357421875, -0.052215576171875, -1.12890625, 0.55517578125, -0.28515625, -0.3603515625, 0.68896484375, 0.67626953125, 0.003070831298828125, 1.2236328125, 0.1597900390625, -1.3076171875, 0.99951171875, -2.5078125, -1.2119140625, 0.1749267578125, -1.1865234375, -1.234375, -0.1180419921875, -1.751953125, 0.033050537109375, 0.234130859375, -3.107421875, -1.0380859375, 0.61181640625, -0.87548828125, 0.3154296875, -1.103515625, 0.261474609375, -1.130859375, -0.7470703125, -0.43408203125, 1.3828125, -0.41259765625, -1.7587890625, 0.765625, 0.004852294921875, 0.135498046875, -0.76953125, -0.1314697265625, 0.400390625, 1.43359375, 0.07135009765625, 0.0645751953125, -0.5869140625, -0.5810546875, -0.2900390625, -1.3037109375, 0.1287841796875, -0.27490234375, 0.59228515625, 2.333984375, -0.54541015625, -0.556640625, 0.447265625, -0.806640625, 0.09149169921875, -0.70654296875, -0.357177734375, -1.099609375, -0.5576171875, -0.44189453125, 0.400390625, -0.666015625, -1.4619140625, 0.728515625, -1.5986328125, 0.153076171875, -0.126708984375, -2.83984375, -1.84375, -0.2469482421875, 0.677734375, 0.43701171875, 3.298828125, 1.1591796875, -0.7158203125, -0.8251953125, 0.451171875, -2.376953125, -0.58642578125, -0.86767578125, 0.0789794921875, 0.1351318359375, -0.325439453125, 0.484375, 1.166015625, -0.1610107421875, -0.15234375, -0.54638671875, -0.806640625, 0.285400390625, 0.1661376953125, -0.50146484375, -1.0478515625, 1.5751953125, 0.0313720703125, 0.2396240234375, -0.6572265625, -0.1258544921875, -1.060546875, 1.3076171875, -0.301513671875, -1.2412109375, 0.6376953125, -1.5693359375, 0.354248046875, 0.2427978515625, -0.392333984375, 0.61962890625, -0.58837890625, -1.71484375, -0.2098388671875, -0.828125, 0.330810546875, 0.16357421875, -0.2259521484375, 0.0972900390625, -0.451416015625, 1.79296875, -1.673828125, -1.58203125, -2.099609375, -0.487548828125, -0.87060546875, 0.62646484375, -1.470703125, -0.1558837890625, 0.4609375, 1.3369140625, 0.2322998046875, 0.1632080078125, 0.65966796875, 1.0810546875, 0.1041259765625, 0.63232421875, -0.32421875, -1.04296875, -1.046875, -1.3720703125, -0.8486328125, 0.1290283203125, 0.137939453125, 0.1549072265625, -1.0908203125, 0.0167694091796875, -0.31689453125, 1.390625, 0.07269287109375, 1.0390625, 1.1162109375, -0.455810546875, -0.06689453125, -0.053741455078125, 0.5048828125, -0.8408203125, -1.19921875, 0.87841796875, 0.7421875, 0.2030029296875, 0.109619140625, -0.59912109375, -1.337890625, -0.74169921875, -0.64453125, -1.326171875, 0.21044921875, -1.3583984375, -1.685546875, -0.472900390625, -0.270263671875, 0.99365234375, -0.96240234375, 1.1279296875, -0.45947265625, -0.45654296875, -0.99169921875, -3.515625, -1.9853515625, 0.73681640625, 0.92333984375, -0.56201171875, -1.4453125, -2.078125, 0.94189453125, -1.333984375, 0.0982666015625, 0.60693359375, 0.367431640625, 3.015625, -1.1357421875, -1.5634765625, 0.90234375, -0.1783447265625, 0.1802978515625, -0.317138671875, -0.513671875, 1.2353515625, -0.033203125, 1.4482421875, 1.0087890625, 0.9248046875, 0.10418701171875, 0.7626953125, -1.3798828125, 0.276123046875, 0.55224609375, 1.1005859375, -0.62158203125, -0.806640625, 0.65087890625, 0.270263671875, -0.339111328125, -0.9384765625, -0.09381103515625, -0.7216796875, 1.37890625, -0.398193359375, -0.3095703125, -1.4912109375, 0.96630859375, 0.43798828125, 0.62255859375, 0.0213470458984375, 0.235595703125, -1.2958984375, 0.0157318115234375, -0.810546875, 1.9736328125, -0.2462158203125, 0.720703125, 0.822265625, -0.755859375, -0.658203125, 0.344482421875, -2.892578125, -0.282470703125, 1.2529296875, -0.294189453125, 0.6748046875, -0.80859375, 0.9287109375, 1.27734375, -1.71875, -0.166015625, 0.47412109375, -0.41259765625, -1.3681640625, -0.978515625, -0.77978515625, -1.044921875, -0.90380859375, -0.08184814453125, -0.86181640625, -0.10772705078125, -0.299560546875, -0.4306640625, -0.47119140625, 0.95703125, 1.107421875, 0.91796875, 0.76025390625, 0.7392578125, -0.09161376953125, -0.7392578125, 0.9716796875, -0.395751953125, -0.75390625, -0.164306640625, -0.087646484375, 0.028564453125, -0.91943359375, -0.66796875, 2.486328125, 0.427734375, 0.626953125, 0.474853515625, 0.0926513671875, 0.830078125, -0.6923828125, 0.7841796875, -0.89208984375, -2.482421875, 0.034912109375, -1.3447265625, -0.475341796875, -0.286376953125, -0.732421875, 0.190673828125, -0.491455078125, -3.091796875, -1.2783203125, -0.66015625, -0.1507568359375, 0.042236328125, -1.025390625, 0.12744140625, -1.984375, -0.393798828125, -1.25, -1.140625, 1.77734375, 0.2457275390625, -0.8017578125, 0.7763671875, -0.387939453125, -0.3662109375, 1.1572265625, 0.123291015625, -0.07135009765625, 1.412109375, -0.685546875, -3.078125, 0.031524658203125, -0.70458984375, 0.78759765625, 0.433837890625, -1.861328125, -1.33203125, 2.119140625, -1.3544921875, -0.6591796875, -1.4970703125, 0.40625, -2.078125, -1.30859375, 0.050262451171875, -0.60107421875, 1.0078125, 0.05657958984375, -0.96826171875, 0.0264892578125, 0.159912109375, 0.84033203125, -1.1494140625, -0.0433349609375, -0.2034912109375, 1.09765625, -1.142578125, -0.283203125, -0.427978515625, 1.0927734375, -0.67529296875, -0.61572265625, 2.517578125, 0.84130859375, 1.8662109375, 0.1748046875, -0.407958984375, -0.029449462890625, -0.27587890625, -0.958984375, -0.10028076171875, 1.248046875, -0.0792236328125, -0.45556640625, 0.7685546875, 1.5556640625, -1.8759765625, -0.131591796875, -1.3583984375, 0.7890625, 0.80810546875, -1.0322265625, -0.53076171875, -0.1484375, -1.7841796875, -1.2470703125, 0.17138671875, -0.04864501953125, -0.80322265625, -0.0933837890625, 0.984375, 0.7001953125, 0.5380859375, 0.2022705078125, -1.1865234375, 0.5439453125, 1.1318359375, 0.79931640625, 0.32666015625, -1.26171875, 0.457763671875, 1.1591796875, -0.34423828125, 0.65771484375, 0.216552734375, 1.19140625, -0.2744140625, -0.020416259765625, -0.86376953125, 0.93017578125, 1.0556640625, 0.69873046875, -0.15087890625, -0.33056640625, 0.8505859375, 0.06890869140625, 0.359375, -0.262939453125, 0.12493896484375, 0.017059326171875, -0.98974609375, 0.5107421875, 0.2408447265625, 0.615234375, -0.62890625, 0.86962890625, -0.07427978515625, 0.85595703125, 0.300537109375, -1.072265625, -1.6064453125, -0.353515625, -0.484130859375, -0.6044921875, -0.455810546875, 0.95849609375, 1.3671875, 0.544921875, 0.560546875, 0.34521484375, -0.6513671875, -0.410400390625, -0.2021484375, -0.1656494140625, 0.073486328125, 0.84716796875, -1.7998046875, -1.0126953125, -0.1324462890625, 0.95849609375, -0.669921875, -0.79052734375, -2.193359375, -0.42529296875, -1.7275390625, -1.04296875, 0.716796875, -0.4423828125, -1.193359375, 0.61572265625, -1.5224609375, 0.62890625, -0.705078125, 0.677734375, -0.213134765625, -1.6748046875, -1.087890625, -0.65185546875, -1.1337890625, 2.314453125, -0.352783203125, -0.27001953125, -2.01953125, -1.2685546875, 0.308837890625, -0.280517578125, -1.3798828125, -1.595703125, 0.642578125, 1.693359375, -0.82470703125, -1.255859375, 0.57373046875, 1.5859375, 1.068359375, -0.876953125, 0.370849609375, 1.220703125, 0.59765625, 0.007602691650390625, 0.09326171875, -0.9521484375, -0.024932861328125, -0.94775390625, -0.299560546875, -0.002536773681640625, 1.41796875, -0.06903076171875, -1.5927734375, 0.353515625, 3.63671875, -0.765625, -1.1142578125, 0.4287109375, -0.86865234375, -0.9267578125, -0.21826171875, -1.10546875, 0.29296875, -0.225830078125, 0.5400390625, -0.45556640625, -0.68701171875, -0.79150390625, -1.0810546875, 0.25439453125, -1.2998046875, -0.494140625, -0.1510009765625, 1.5615234375, -0.4248046875, -0.486572265625, 0.45458984375, 0.047637939453125, -0.11639404296875, 0.057403564453125, 0.130126953125, -0.10125732421875, -0.56201171875, 1.4765625, -1.7451171875, 1.34765625, -0.45703125, 0.873046875, -0.056121826171875, -0.8876953125, -0.986328125, 1.5654296875, 0.49853515625, 0.55859375, -0.2198486328125, 0.62548828125, 0.2734375, -0.63671875, -0.41259765625, -1.2705078125, 0.0665283203125, 1.3369140625, 0.90283203125, -0.77685546875, -1.5, -1.8525390625, -1.314453125, -0.86767578125, -0.331787109375, 0.1590576171875, 0.94775390625, -0.1771240234375, 1.638671875, -2.17578125, 0.58740234375, 0.424560546875, -0.3466796875, 0.642578125, 0.473388671875, 0.96435546875, 1.38671875, -0.91357421875, 1.0361328125, -0.67333984375, 1.5009765625]]]).to(device) + + cond = [[prompt_embeds, {}]] + + return io.NodeOutput(cond) + + +class LotusExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + LotusConditioning, + ] + + +async def comfy_entrypoint() -> LotusExtension: + return LotusExtension() diff --git a/ComfyUI/comfy_extras/nodes_lt.py b/ComfyUI/comfy_extras/nodes_lt.py new file mode 100644 index 0000000000000000000000000000000000000000..b4806bd6a2d381cf42550f54b42972cd61b49493 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_lt.py @@ -0,0 +1,920 @@ +import nodes +import node_helpers +import torch +import torchaudio +import comfy.model_management +import comfy.model_sampling +import comfy.samplers +import comfy.utils +import math +import numpy as np +import av +from io import BytesIO +from typing_extensions import override +from comfy.ldm.lightricks.symmetric_patchifier import SymmetricPatchifier, latent_to_pixel_coords +from comfy_api.latest import ComfyExtension, io + +ICLoRAParameters = io.Custom("IC_LORA_PARAMETERS") + + +class GetICLoRAParameters(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="GetICLoRAParameters", + display_name="Get IC-LoRA Parameters", + description="Extracts IC-LoRA parameters from the safetensors metadata of a LoRA-loaded " + "model and outputs them for LTXVAddGuide (eg. reference_downscale_factor).", + category="model/conditioning/ltxv", + search_aliases=["ic-lora", "ic lora", "iclora", "downscale factor", "reference downscale"], + inputs=[ + io.Model.Input( + "iclora_model", + tooltip="Direct output from a LoRA Loader for the specific IC-LoRA " + "from which to extract the metadata.", + ), + ], + outputs=[ + ICLoRAParameters.Output( + "iclora_parameters", + tooltip="IC-LoRA parameters extracted from the LoRA metadata " + "(eg. reference_downscale_factor). Connect to LTXVAddGuide " + "if the LoRA requires special handling of the guides.", + ), + ], + ) + + @classmethod + def execute(cls, iclora_model) -> io.NodeOutput: + metadata = iclora_model.get_attachment("lora_metadata") + factor = 1 + if metadata: + try: + factor = max(1, round(float(next(v for k, v in metadata.items() if k.endswith("reference_downscale_factor"))))) + except (StopIteration, TypeError, ValueError): + factor = 1 + parameters = {"reference_downscale_factor": factor} + return io.NodeOutput(parameters) + + +class EmptyLTXVLatentVideo(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="EmptyLTXVLatentVideo", + category="model/latent/ltxv", + inputs=[ + io.Int.Input("width", default=768, min=64, max=nodes.MAX_RESOLUTION, step=32), + io.Int.Input("height", default=512, min=64, max=nodes.MAX_RESOLUTION, step=32), + io.Int.Input("length", default=97, min=1, max=nodes.MAX_RESOLUTION, step=8), + io.Int.Input("batch_size", default=1, min=1, max=4096), + ], + outputs=[ + io.Latent.Output(), + ], + ) + + @classmethod + def execute(cls, width, height, length, batch_size=1) -> io.NodeOutput: + latent = torch.zeros([batch_size, 128, ((length - 1) // 8) + 1, height // 32, width // 32], device=comfy.model_management.intermediate_device()) + return io.NodeOutput({"samples": latent, "downscale_ratio_spacial": 32}) + + generate = execute # TODO: remove + +class LTXVImgToVideo(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LTXVImgToVideo", + category="model/conditioning/ltxv", + inputs=[ + io.Conditioning.Input("positive"), + io.Conditioning.Input("negative"), + io.Vae.Input("vae"), + io.Image.Input("image"), + io.Int.Input("width", default=768, min=64, max=nodes.MAX_RESOLUTION, step=32), + io.Int.Input("height", default=512, min=64, max=nodes.MAX_RESOLUTION, step=32), + io.Int.Input("length", default=97, min=9, max=nodes.MAX_RESOLUTION, step=8), + io.Int.Input("batch_size", default=1, min=1, max=4096), + io.Float.Input("strength", default=1.0, min=0.0, max=1.0), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + io.Latent.Output(display_name="latent"), + ], + ) + + @classmethod + def execute(cls, positive, negative, image, vae, width, height, length, batch_size, strength) -> io.NodeOutput: + pixels = comfy.utils.common_upscale(image.movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1) + encode_pixels = pixels[:, :, :, :3] + t = vae.encode(encode_pixels) + + latent = torch.zeros([batch_size, 128, ((length - 1) // 8) + 1, height // 32, width // 32], device=comfy.model_management.intermediate_device()) + latent[:, :, :t.shape[2]] = t + + conditioning_latent_frames_mask = torch.ones( + (batch_size, 1, latent.shape[2], 1, 1), + dtype=torch.float32, + device=latent.device, + ) + conditioning_latent_frames_mask[:, :, :t.shape[2]] = 1.0 - strength + + return io.NodeOutput(positive, negative, {"samples": latent, "noise_mask": conditioning_latent_frames_mask}) + + generate = execute # TODO: remove + + +class LTXVImgToVideoInplace(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LTXVImgToVideoInplace", + category="model/conditioning/ltxv", + inputs=[ + io.Vae.Input("vae"), + io.Image.Input("image"), + io.Latent.Input("latent"), + io.Float.Input("strength", default=1.0, min=0.0, max=1.0), + io.Boolean.Input("bypass", default=False, tooltip="Bypass the conditioning.") + ], + outputs=[ + io.Latent.Output(display_name="latent"), + ], + ) + + @classmethod + def execute(cls, vae, image, latent, strength, bypass=False) -> io.NodeOutput: + if bypass: + return (latent,) + + samples = latent["samples"].clone() + _, height_scale_factor, width_scale_factor = ( + vae.downscale_index_formula + ) + + _, _, _, latent_height, latent_width = samples.shape + width = latent_width * width_scale_factor + height = latent_height * height_scale_factor + + if image.shape[1] != height or image.shape[2] != width: + pixels = comfy.utils.common_upscale(image.movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1) + else: + pixels = image + encode_pixels = pixels[:, :, :, :3] + t = vae.encode(encode_pixels) + + samples[:, :, :t.shape[2]] = t + + conditioning_latent_frames_mask = get_noise_mask(latent) + conditioning_latent_frames_mask[:, :, :t.shape[2]] = 1.0 - strength + + return io.NodeOutput({"samples": samples, "noise_mask": conditioning_latent_frames_mask}) + + generate = execute # TODO: remove + + +def _append_guide_attention_entry(positive, negative, pre_filter_count, latent_shape, strength=1.0, attention_mask=None): + """Append a guide_attention_entry to both positive and negative conditioning. + + Each entry tracks one guide reference for per-reference attention control. + Entries are derived independently from each conditioning to avoid cross-contamination. + """ + new_entry = { + "pre_filter_count": pre_filter_count, + "strength": strength, + "pixel_mask": attention_mask.unsqueeze(0).unsqueeze(0) if attention_mask is not None else None, # reshape to (1, 1, F, H, W) + "latent_shape": latent_shape, + } + + results = [] + for cond in (positive, negative): + # Read existing entries from this specific conditioning + existing = [] + for t in cond: + found = t[1].get("guide_attention_entries", None) + if found is not None: + existing = found + break + # Shallow copy only and append (pixel_mask is never mutated). + entries = [*existing, new_entry] + results.append(node_helpers.conditioning_set_values( + cond, {"guide_attention_entries": entries} + )) + return results[0], results[1] + + +def conditioning_get_any_value(conditioning, key, default=None): + for t in conditioning: + if key in t[1]: + return t[1][key] + return default + + +def get_noise_mask(latent): + noise_mask = latent.get("noise_mask", None) + latent_image = latent["samples"] + if noise_mask is None: + batch_size, _, latent_length, _, _ = latent_image.shape + noise_mask = torch.ones( + (batch_size, 1, latent_length, 1, 1), + dtype=torch.float32, + device=latent_image.device, + ) + else: + noise_mask = noise_mask.clone() + return noise_mask + +def get_keyframe_idxs(cond, latent_shape=None): + keyframe_idxs = conditioning_get_any_value(cond, "keyframe_idxs", None) + if keyframe_idxs is None: + return None, 0 + # Get number of keyframes from latent_shape or guide_attention_entries if available + if latent_shape is not None and len(latent_shape) == 5: + tokens_per_frame = latent_shape[-2] * latent_shape[-1] + num_keyframes = keyframe_idxs.shape[2] // tokens_per_frame + return keyframe_idxs, num_keyframes + entries = conditioning_get_any_value(cond, "guide_attention_entries", None) + if entries: + num_keyframes = sum(e["latent_shape"][0] for e in entries) + return keyframe_idxs, num_keyframes + # fallback, may under-count if keyframes share t-start + # keyframe_idxs contains start/end positions (last dimension), checking for unqiue values only for start + num_keyframes = torch.unique(keyframe_idxs[:, 0, :, 0]).shape[0] + return keyframe_idxs, num_keyframes + +class LTXVAddGuide(io.ComfyNode): + PATCHIFIER = SymmetricPatchifier(1, start_end=True) + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LTXVAddGuide", + category="model/conditioning/ltxv", + inputs=[ + io.Conditioning.Input("positive"), + io.Conditioning.Input("negative"), + io.Vae.Input("vae"), + io.Latent.Input("latent"), + io.Image.Input( + "image", + tooltip="Image or video to condition the latent video on. Must be 8*n + 1 frames. " + "If the video is not 8*n + 1 frames, it will be cropped to the nearest 8*n + 1 frames.", + ), + io.Int.Input( + "frame_idx", + default=0, + min=-9999, + max=9999, + tooltip="Frame index to start the conditioning at. " + "For single-frame images or videos with 1-8 frames, any frame_idx value is acceptable. " + "For videos with 9+ frames, frame_idx must be divisible by 8, otherwise it will be rounded " + "down to the nearest multiple of 8. Negative values are counted from the end of the video.", + ), + io.Float.Input("strength", default=1.0, min=0.0, max=10.0, step=0.01), + io.Mask.Input( + "attention_mask", + optional=True, + tooltip="Optional pixel-space spatial mask. Controls per-region " + "conditioning influence via self-attention, multiplied by strength.", + ), + ICLoRAParameters.Input( + "iclora_parameters", + optional=True, + tooltip="Optional IC-LoRA parameters from a Get IC-LoRA Parameters node. " + "Used for adjusting guide processing as required by certain IC-LoRAs " + "(eg. those with a reference_downscale_factor > 1). " + "When chained, each LTXVAddGuide uses only the parameters connected to it.", + ), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + io.Latent.Output(display_name="latent"), + ], + ) + + @classmethod + def encode(cls, vae, latent_width, latent_height, images, scale_factors, latent_downscale_factor=1): + time_scale_factor, width_scale_factor, height_scale_factor = scale_factors + images = images[:(images.shape[0] - 1) // time_scale_factor * time_scale_factor + 1] + target_width = int(latent_width * width_scale_factor / latent_downscale_factor) + target_height = int(latent_height * height_scale_factor / latent_downscale_factor) + pixels = comfy.utils.common_upscale(images.movedim(-1, 1), target_width, target_height, "bilinear", crop="center").movedim(1, -1) + encode_pixels = pixels[:, :, :, :3] + t = vae.encode(encode_pixels) + return encode_pixels, t + + @classmethod + def dilate_latent(cls, guide_latent, latent_downscale_factor): + if latent_downscale_factor <= 1: + return guide_latent, None + scale = int(latent_downscale_factor) + dilated_shape = guide_latent.shape[:3] + (guide_latent.shape[3] * scale, guide_latent.shape[4] * scale) + dilated = torch.zeros(dilated_shape, device=guide_latent.device, dtype=guide_latent.dtype) + dilated[..., ::scale, ::scale] = guide_latent + dilated_mask = torch.full( + (dilated.shape[0], 1, dilated.shape[2], dilated.shape[3], dilated.shape[4]), + -1.0, device=guide_latent.device, dtype=guide_latent.dtype, + ) + dilated_mask[..., ::scale, ::scale] = 1.0 + return dilated, dilated_mask + + @classmethod + def get_reference_downscale_factor(cls, iclora_parameters): + if not iclora_parameters: + return 1 + try: + factor = max(1, round(float(iclora_parameters.get("reference_downscale_factor", 1)))) + except (TypeError, ValueError): + factor = 1 + return factor + + @classmethod + def get_latent_index(cls, cond, latent_length, guide_length, frame_idx, scale_factors, latent_shape=None): + time_scale_factor, _, _ = scale_factors + _, num_keyframes = get_keyframe_idxs(cond, latent_shape) + latent_count = latent_length - num_keyframes + frame_idx = frame_idx if frame_idx >= 0 else max((latent_count - 1) * time_scale_factor + 1 + frame_idx, 0) + if guide_length > 1 and frame_idx != 0: + frame_idx = (frame_idx - 1) // time_scale_factor * time_scale_factor + 1 # frame index - 1 must be divisible by 8 or frame_idx == 0 + + latent_idx = (frame_idx + time_scale_factor - 1) // time_scale_factor + + return frame_idx, latent_idx + + @classmethod + def add_keyframe_index(cls, cond, frame_idx, guiding_latent, scale_factors, latent_downscale_factor=1, causal_fix=None): + keyframe_idxs, _ = get_keyframe_idxs(cond) + _, latent_coords = cls.PATCHIFIER.patchify(guiding_latent) + if causal_fix is None: + causal_fix = frame_idx == 0 or guiding_latent.shape[2] == 1 + pixel_coords = latent_to_pixel_coords(latent_coords, scale_factors, causal_fix=causal_fix) + pixel_coords[:, 0] += frame_idx + + # The following adjusts keyframe end positions for small grid IC-LoRA. + # After dilation, the small grid has the same size and position as the large grid, + # but each token encodes a larger image patch. We adjust the end position (not start) + # so that RoPE represents the correct middle point of each token. + # keyframe_idxs dims: (batch, spatial_dim [t,h,w], token_id, [start, end]) + # We only adjust h,w (not t) in dim 1, and only end (not start) in dim 3. + spatial_end_offset = (latent_downscale_factor - 1) * torch.tensor( + scale_factors[1:], + device=pixel_coords.device, + ).view(1, -1, 1, 1) + pixel_coords[:, 1:, :, 1:] += spatial_end_offset.to(pixel_coords.dtype) + + if keyframe_idxs is None: + keyframe_idxs = pixel_coords + else: + keyframe_idxs = torch.cat([keyframe_idxs, pixel_coords], dim=2) + return node_helpers.conditioning_set_values(cond, {"keyframe_idxs": keyframe_idxs}) + + @classmethod + def append_keyframe(cls, positive, negative, frame_idx, latent_image, noise_mask, guiding_latent, strength, scale_factors, guide_mask=None, in_channels=128, latent_downscale_factor=1, causal_fix=None): + if latent_image.shape[1] != in_channels or guiding_latent.shape[1] != in_channels: + raise ValueError("Adding guide to a combined AV latent is not supported.") + + positive = cls.add_keyframe_index(positive, frame_idx, guiding_latent, scale_factors, latent_downscale_factor, causal_fix=causal_fix) + negative = cls.add_keyframe_index(negative, frame_idx, guiding_latent, scale_factors, latent_downscale_factor, causal_fix=causal_fix) + + if guide_mask is not None: + target_h = max(noise_mask.shape[3], guide_mask.shape[3]) + target_w = max(noise_mask.shape[4], guide_mask.shape[4]) + + if noise_mask.shape[3] == 1 or noise_mask.shape[4] == 1: + noise_mask = noise_mask.expand(-1, -1, -1, target_h, target_w) + + if guide_mask.shape[3] == 1 or guide_mask.shape[4] == 1: + guide_mask = guide_mask.expand(-1, -1, -1, target_h, target_w) + mask = guide_mask - strength + else: + mask = torch.full( + (noise_mask.shape[0], 1, guiding_latent.shape[2], noise_mask.shape[3], noise_mask.shape[4]), + max(0.0, 1.0 - strength), # clamp here to amplify only via the attention mask + dtype=noise_mask.dtype, + device=noise_mask.device, + ) + # This solves audio video combined latent case where latent_image has audio latent concatenated + # in channel dimension with video latent. The solution is to pad guiding latent accordingly. + if latent_image.shape[1] > guiding_latent.shape[1]: + pad_len = latent_image.shape[1] - guiding_latent.shape[1] + guiding_latent = torch.nn.functional.pad(guiding_latent, pad=(0, 0, 0, 0, 0, 0, 0, pad_len), value=0) + latent_image = torch.cat([latent_image, guiding_latent], dim=2) + noise_mask = torch.cat([noise_mask, mask], dim=2) + return positive, negative, latent_image, noise_mask + + @classmethod + def replace_latent_frames(cls, latent_image, noise_mask, guiding_latent, latent_idx, strength): + cond_length = guiding_latent.shape[2] + assert latent_image.shape[2] >= latent_idx + cond_length, "Conditioning frames exceed the length of the latent sequence." + + mask = torch.full( + (noise_mask.shape[0], 1, cond_length, 1, 1), + max(0.0, 1.0 - strength), # clamp here to amplify only via the attention mask + dtype=noise_mask.dtype, + device=noise_mask.device, + ) + + latent_image = latent_image.clone() + noise_mask = noise_mask.clone() + + latent_image[:, :, latent_idx : latent_idx + cond_length] = guiding_latent + noise_mask[:, :, latent_idx : latent_idx + cond_length] = mask + + return latent_image, noise_mask + + @classmethod + def execute(cls, positive, negative, vae, latent, image, frame_idx, strength, attention_mask=None, iclora_parameters=None) -> io.NodeOutput: + scale_factors = vae.downscale_index_formula + latent_image = latent["samples"] + noise_mask = get_noise_mask(latent) + + _, _, latent_length, latent_height, latent_width = latent_image.shape + + latent_downscale_factor = cls.get_reference_downscale_factor(iclora_parameters) + if latent_downscale_factor > 1: + if latent_width % latent_downscale_factor != 0 or latent_height % latent_downscale_factor != 0: + raise ValueError( + f"Latent spatial size {latent_width}x{latent_height} must be divisible by " + f"reference_downscale_factor {latent_downscale_factor} from the IC-LoRA parameters." + ) + + # For mid-video multi-frame guides, prepend+strip a throwaway first frame so the VAE's "first latent = 1 pixel frame" asymmetry lands on the discarded slot + time_scale_factor = scale_factors[0] + num_frames_to_keep = ((image.shape[0] - 1) // time_scale_factor) * time_scale_factor + 1 + resolved_frame_idx = frame_idx + if frame_idx < 0: + _, num_keyframes = get_keyframe_idxs(positive, latent_image.shape) + resolved_frame_idx = max((latent_length - num_keyframes - 1) * time_scale_factor + 1 + frame_idx, 0) + causal_fix = resolved_frame_idx == 0 or num_frames_to_keep == 1 + + if not causal_fix: + image = torch.cat([image[:1], image], dim=0) + + image, t = cls.encode(vae, latent_width, latent_height, image, scale_factors, latent_downscale_factor) + + if not causal_fix: + t = t[:, :, 1:, :, :] + image = image[1:] + + guide_latent_shape = list(t.shape[2:]) # pre-dilation [F, H, W] for spatial-mask downsampling + guide_mask = None + if latent_downscale_factor > 1: + t, guide_mask = cls.dilate_latent(t, latent_downscale_factor) + + frame_idx, latent_idx = cls.get_latent_index(positive, latent_length, len(image), frame_idx, scale_factors, latent_shape=latent_image.shape) + assert latent_idx + t.shape[2] <= latent_length, "Conditioning frames exceed the length of the latent sequence." + + positive, negative, latent_image, noise_mask = cls.append_keyframe( + positive, + negative, + frame_idx, + latent_image, + noise_mask, + t, + strength, + scale_factors, + guide_mask=guide_mask, + latent_downscale_factor=latent_downscale_factor, + causal_fix=causal_fix, + ) + + # Track this guide for per-reference attention control. + pre_filter_count = t.shape[2] * t.shape[3] * t.shape[4] + positive, negative = _append_guide_attention_entry( + positive, negative, pre_filter_count, guide_latent_shape, strength=strength, + attention_mask=attention_mask, + ) + + return io.NodeOutput(positive, negative, {"samples": latent_image, "noise_mask": noise_mask}) + + generate = execute # TODO: remove + + +class LTXVCropGuides(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LTXVCropGuides", + category="model/conditioning/ltxv", + inputs=[ + io.Conditioning.Input("positive"), + io.Conditioning.Input("negative"), + io.Latent.Input("latent"), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + io.Latent.Output(display_name="latent"), + ], + ) + + @classmethod + def execute(cls, positive, negative, latent) -> io.NodeOutput: + latent_image = latent["samples"].clone() + noise_mask = get_noise_mask(latent) + + _, num_keyframes = get_keyframe_idxs(positive, latent_image.shape) + if num_keyframes == 0: + return io.NodeOutput(positive, negative, {"samples": latent_image, "noise_mask": noise_mask},) + + latent_image = latent_image[:, :, :-num_keyframes] + noise_mask = noise_mask[:, :, :-num_keyframes] + + positive = node_helpers.conditioning_set_values(positive, { + "keyframe_idxs": None, + "guide_attention_entries": None, + }) + negative = node_helpers.conditioning_set_values(negative, { + "keyframe_idxs": None, + "guide_attention_entries": None, + }) + + return io.NodeOutput(positive, negative, {"samples": latent_image, "noise_mask": noise_mask}) + + crop = execute # TODO: remove + + +class LTXVConditioning(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LTXVConditioning", + category="model/conditioning/ltxv", + inputs=[ + io.Conditioning.Input("positive"), + io.Conditioning.Input("negative"), + io.Float.Input("frame_rate", default=25.0, min=0.0, max=1000.0, step=0.01), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + ], + ) + + @classmethod + def execute(cls, positive, negative, frame_rate) -> io.NodeOutput: + positive = node_helpers.conditioning_set_values(positive, {"frame_rate": frame_rate}) + negative = node_helpers.conditioning_set_values(negative, {"frame_rate": frame_rate}) + return io.NodeOutput(positive, negative) + + +class ModelSamplingLTXV(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ModelSamplingLTXV", + category="model/patch/ltxv", + inputs=[ + io.Model.Input("model"), + io.Float.Input("max_shift", default=2.05, min=0.0, max=100.0, step=0.01), + io.Float.Input("base_shift", default=0.95, min=0.0, max=100.0, step=0.01), + io.Latent.Input("latent", optional=True), + ], + outputs=[ + io.Model.Output(), + ], + ) + + @classmethod + def execute(cls, model, max_shift, base_shift, latent=None) -> io.NodeOutput: + m = model.clone() + + if latent is None: + tokens = 4096 + else: + tokens = math.prod(latent["samples"].shape[2:]) + + x1 = 1024 + x2 = 4096 + mm = (max_shift - base_shift) / (x2 - x1) + b = base_shift - mm * x1 + shift = (tokens) * mm + b + + sampling_base = comfy.model_sampling.ModelSamplingFlux + sampling_type = comfy.model_sampling.CONST + + class ModelSamplingAdvanced(sampling_base, sampling_type): + pass + + model_sampling = ModelSamplingAdvanced(model.model.model_config) + model_sampling.set_parameters(shift=shift) + m.add_object_patch("model_sampling", model_sampling) + + return io.NodeOutput(m) + + +class LTXVScheduler(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LTXVScheduler", + category="model/sampling/schedulers", + inputs=[ + io.Int.Input("steps", default=20, min=1, max=10000), + io.Float.Input("max_shift", default=2.05, min=0.0, max=100.0, step=0.01), + io.Float.Input("base_shift", default=0.95, min=0.0, max=100.0, step=0.01), + io.Boolean.Input( + id="stretch", + default=True, + tooltip="Stretch the sigmas to be in the range [terminal, 1].", + advanced=True, + ), + io.Float.Input( + id="terminal", + default=0.1, + min=0.0, + max=0.99, + step=0.01, + tooltip="The terminal value of the sigmas after stretching.", + advanced=True, + ), + io.Latent.Input("latent", optional=True), + ], + outputs=[ + io.Sigmas.Output(), + ], + ) + + @classmethod + def execute(cls, steps, max_shift, base_shift, stretch, terminal, latent=None) -> io.NodeOutput: + if latent is None: + tokens = 4096 + else: + tokens = math.prod(latent["samples"].shape[2:]) + + sigmas = torch.linspace(1.0, 0.0, steps + 1) + + x1 = 1024 + x2 = 4096 + mm = (max_shift - base_shift) / (x2 - x1) + b = base_shift - mm * x1 + sigma_shift = (tokens) * mm + b + + power = 1 + sigmas = torch.where( + sigmas != 0, + math.exp(sigma_shift) / (math.exp(sigma_shift) + (1 / sigmas - 1) ** power), + 0, + ) + + # Stretch sigmas so that its final value matches the given terminal value. + if stretch: + non_zero_mask = sigmas != 0 + non_zero_sigmas = sigmas[non_zero_mask] + one_minus_z = 1.0 - non_zero_sigmas + scale_factor = one_minus_z[-1] / (1.0 - terminal) + stretched = 1.0 - (one_minus_z / scale_factor) + sigmas[non_zero_mask] = stretched + + return io.NodeOutput(sigmas) + +def encode_single_frame(output_file, image_array: np.ndarray, crf): + container = av.open(output_file, "w", format="mp4") + try: + stream = container.add_stream( + "libx264", rate=1, options={"crf": str(crf), "preset": "veryfast"} + ) + stream.height = image_array.shape[0] + stream.width = image_array.shape[1] + av_frame = av.VideoFrame.from_ndarray(image_array, format="rgb24").reformat( + format="yuv420p" + ) + container.mux(stream.encode(av_frame)) + container.mux(stream.encode()) + finally: + container.close() + + +def decode_single_frame(video_file): + container = av.open(video_file) + try: + stream = next(s for s in container.streams if s.type == "video") + frame = next(container.decode(stream)) + finally: + container.close() + return frame.to_ndarray(format="rgb24") + + +def preprocess(image: torch.Tensor, crf=29): + if crf == 0: + return image + + image_array = (image[:(image.shape[0] // 2) * 2, :(image.shape[1] // 2) * 2] * 255.0).byte().cpu().numpy() + with BytesIO() as output_file: + encode_single_frame(output_file, image_array, crf) + video_bytes = output_file.getvalue() + with BytesIO(video_bytes) as video_file: + image_array = decode_single_frame(video_file) + tensor = torch.tensor(image_array, dtype=image.dtype, device=image.device) / 255.0 + return tensor + + +class LTXVPreprocess(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LTXVPreprocess", + display_name="LTXV Preprocess", + category="video/preprocessors", + inputs=[ + io.Image.Input("image"), + io.Int.Input( + id="img_compression", default=35, min=0, max=100, tooltip="Amount of compression to apply on image." + ), + ], + outputs=[ + io.Image.Output(display_name="output_image"), + ], + ) + + @classmethod + def execute(cls, image, img_compression) -> io.NodeOutput: + output_images = [] + for i in range(image.shape[0]): + output_images.append(preprocess(image[i], img_compression)) + return io.NodeOutput(torch.stack(output_images)) + + preprocess = execute # TODO: remove + + +import comfy.nested_tensor +class LTXVConcatAVLatent(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LTXVConcatAVLatent", + display_name="Concat AV Latent", + description="Merge a video latent and an audio latent into a joint AV latent (any AV model, e.g. LTXV or MiniMax H3).", + category="model/latent/ltxv", + inputs=[ + io.Latent.Input("video_latent"), + io.Latent.Input("audio_latent"), + ], + outputs=[ + io.Latent.Output(display_name="latent"), + ], + ) + + @classmethod + def execute(cls, video_latent, audio_latent) -> io.NodeOutput: + output = {} + output.update(video_latent) + output.update(audio_latent) + video_noise_mask = video_latent.get("noise_mask", None) + audio_noise_mask = audio_latent.get("noise_mask", None) + + if video_noise_mask is not None or audio_noise_mask is not None: + if video_noise_mask is None: + video_noise_mask = torch.ones_like(video_latent["samples"]) + if audio_noise_mask is None: + audio_noise_mask = torch.ones_like(audio_latent["samples"]) + output["noise_mask"] = comfy.nested_tensor.NestedTensor((video_noise_mask, audio_noise_mask)) + + output["samples"] = comfy.nested_tensor.NestedTensor((video_latent["samples"], audio_latent["samples"])) + + return io.NodeOutput(output) + + +class LTXVSeparateAVLatent(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LTXVSeparateAVLatent", + display_name="Separate AV Latent", + category="model/latent/ltxv", + description="Split a joint AV latent into its video and audio latents (any AV model, e.g. LTXV or MiniMax H3).", + inputs=[ + io.Latent.Input("av_latent"), + ], + outputs=[ + io.Latent.Output(display_name="video_latent"), + io.Latent.Output(display_name="audio_latent"), + ], + ) + + @classmethod + def execute(cls, av_latent) -> io.NodeOutput: + latents = av_latent["samples"].unbind() + video_latent = av_latent.copy() + video_latent["samples"] = latents[0] + audio_latent = av_latent.copy() + audio_latent["samples"] = latents[1] + if "noise_mask" in av_latent: + masks = av_latent["noise_mask"] + if masks is not None: + masks = masks.unbind() + video_latent["noise_mask"] = masks[0] + audio_latent["noise_mask"] = masks[1] + return io.NodeOutput(video_latent, audio_latent) + + +class LTXVReferenceAudio(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="LTXVReferenceAudio", + display_name="LTXV Reference Audio (ID-LoRA)", + category="model/conditioning/ltxv", + description="Set reference audio for ID-LoRA speaker identity transfer. Encodes a reference audio clip into the conditioning and optionally patches the model with identity guidance (extra forward pass without reference, amplifying the speaker identity effect).", + inputs=[ + io.Model.Input("model"), + io.Conditioning.Input("positive"), + io.Conditioning.Input("negative"), + io.Audio.Input("reference_audio", tooltip="Reference audio clip whose speaker identity to transfer. ~5 seconds recommended (training duration). Shorter or longer clips may degrade voice identity transfer."), + io.Vae.Input(id="audio_vae", display_name="Audio VAE", tooltip="LTXV Audio VAE for encoding."), + io.Float.Input("identity_guidance_scale", default=3.0, min=0.0, max=100.0, step=0.01, round=0.01, tooltip="Strength of identity guidance. Runs an extra forward pass without reference each step to amplify speaker identity. Set to 0 to disable (no extra pass)."), + io.Float.Input("start_percent", default=0.0, min=0.0, max=1.0, step=0.001, advanced=True, tooltip="Start of the sigma range where identity guidance is active."), + io.Float.Input("end_percent", default=1.0, min=0.0, max=1.0, step=0.001, advanced=True, tooltip="End of the sigma range where identity guidance is active."), + ], + outputs=[ + io.Model.Output(), + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + ], + ) + + @classmethod + def execute(cls, model, positive, negative, reference_audio, audio_vae, identity_guidance_scale, start_percent, end_percent) -> io.NodeOutput: + # Encode reference audio to latents and patchify + sample_rate = reference_audio["sample_rate"] + vae_sample_rate = getattr(audio_vae, "audio_sample_rate", 44100) + if vae_sample_rate != sample_rate: + waveform = torchaudio.functional.resample(reference_audio["waveform"], sample_rate, vae_sample_rate) + else: + waveform = reference_audio["waveform"] + + audio_latents = audio_vae.encode(waveform.movedim(1, -1)) + b, c, t, f = audio_latents.shape + ref_tokens = audio_latents.permute(0, 2, 1, 3).reshape(b, t, c * f) + ref_audio = {"tokens": ref_tokens} + + positive = node_helpers.conditioning_set_values(positive, {"ref_audio": ref_audio}) + negative = node_helpers.conditioning_set_values(negative, {"ref_audio": ref_audio}) + + # Patch model with identity guidance + m = model.clone() + scale = identity_guidance_scale + model_sampling = m.get_model_object("model_sampling") + sigma_start = model_sampling.percent_to_sigma(start_percent) + sigma_end = model_sampling.percent_to_sigma(end_percent) + + def post_cfg_function(args): + if scale == 0: + return args["denoised"] + + sigma = args["sigma"] + sigma_ = sigma[0].item() + if sigma_ > sigma_start or sigma_ < sigma_end: + return args["denoised"] + + cond_pred = args["cond_denoised"] + cond = args["cond"] + cfg_result = args["denoised"] + model_options = args["model_options"].copy() + x = args["input"] + + # Strip ref_audio from conditioning for the no-reference pass + noref_cond = [] + for entry in cond: + new_entry = entry.copy() + mc = new_entry.get("model_conds", {}).copy() + mc.pop("ref_audio", None) + new_entry["model_conds"] = mc + noref_cond.append(new_entry) + + (pred_noref,) = comfy.samplers.calc_cond_batch( + args["model"], [noref_cond], x, sigma, model_options + ) + + return cfg_result + (cond_pred - pred_noref) * scale + + m.set_model_sampler_post_cfg_function(post_cfg_function) + + return io.NodeOutput(m, positive, negative) + + +class LtxvExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + EmptyLTXVLatentVideo, + LTXVImgToVideo, + LTXVImgToVideoInplace, + ModelSamplingLTXV, + LTXVConditioning, + LTXVScheduler, + GetICLoRAParameters, + LTXVAddGuide, + LTXVPreprocess, + LTXVCropGuides, + LTXVConcatAVLatent, + LTXVSeparateAVLatent, + LTXVReferenceAudio, + ] + + +async def comfy_entrypoint() -> LtxvExtension: + return LtxvExtension() diff --git a/ComfyUI/comfy_extras/nodes_lt_audio.py b/ComfyUI/comfy_extras/nodes_lt_audio.py new file mode 100644 index 0000000000000000000000000000000000000000..3ff18d8d4643004702ce9b3150243c3e11688f08 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_lt_audio.py @@ -0,0 +1,222 @@ +import folder_paths +import comfy.utils +import comfy.model_management +import torch + +from comfy_api.latest import ComfyExtension, io +from comfy_extras.nodes_audio import VAEEncodeAudio + +class LTXVAudioVAELoader(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="LTXVAudioVAELoader", + display_name="Load LTXV Audio VAE", + category="model/loaders", + inputs=[ + io.Combo.Input( + "ckpt_name", + options=folder_paths.get_filename_list("checkpoints"), + tooltip="Audio VAE checkpoint to load.", + ) + ], + outputs=[io.Vae.Output(display_name="Audio VAE")], + ) + + @classmethod + def execute(cls, ckpt_name: str) -> io.NodeOutput: + ckpt_path = folder_paths.get_full_path_or_raise("checkpoints", ckpt_name) + sd, metadata = comfy.utils.load_torch_file(ckpt_path, return_metadata=True) + sd = comfy.utils.state_dict_prefix_replace(sd, {"audio_vae.": "autoencoder.", "vocoder.": "vocoder."}, filter_keys=True) + vae = comfy.sd.VAE(sd=sd, metadata=metadata) + vae.throw_exception_if_invalid() + + return io.NodeOutput(vae) + + +class LTXVAudioVAEEncode(VAEEncodeAudio): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="LTXVAudioVAEEncode", + display_name="LTXV Audio VAE Encode", + category="model/latent/ltxv", + inputs=[ + io.Audio.Input("audio", tooltip="The audio to be encoded."), + io.Vae.Input( + id="audio_vae", + display_name="Audio VAE", + tooltip="The Audio VAE model to use for encoding.", + ), + ], + outputs=[io.Latent.Output(display_name="Audio Latent")], + ) + + @classmethod + def execute(cls, audio, audio_vae) -> io.NodeOutput: + return super().execute(audio_vae, audio) + + +class LTXVAudioVAEDecode(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="LTXVAudioVAEDecode", + display_name="LTXV Audio VAE Decode", + category="model/latent/ltxv", + inputs=[ + io.Latent.Input("samples", tooltip="The latent to be decoded."), + io.Vae.Input( + id="audio_vae", + display_name="Audio VAE", + tooltip="The Audio VAE model used for decoding the latent.", + ), + ], + outputs=[io.Audio.Output(display_name="Audio")], + ) + + @classmethod + def execute(cls, samples, audio_vae) -> io.NodeOutput: + audio_latent = samples["samples"] + if audio_latent.is_nested: + audio_latent = audio_latent.unbind()[-1] + audio = audio_vae.decode(audio_latent).movedim(-1, 1).to(audio_latent.device) + output_audio_sample_rate = audio_vae.first_stage_model.output_sample_rate + return io.NodeOutput( + { + "waveform": audio, + "sample_rate": int(output_audio_sample_rate), + } + ) + + +class LTXVEmptyLatentAudio(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="LTXVEmptyLatentAudio", + display_name="LTXV Empty Latent Audio", + category="model/latent/ltxv", + inputs=[ + io.Int.Input( + "frames_number", + default=97, + min=1, + max=1000, + step=1, + display_mode=io.NumberDisplay.number, + tooltip="Number of frames.", + ), + io.MultiType.Input( + io.Float.Input( + "frame_rate", + default=25.0, + min=1.0, + max=1000.0, + step=0.01, + display_mode=io.NumberDisplay.number, + tooltip="Number of frames per second.", + ), + [io.Int], + ), + io.Int.Input( + "batch_size", + default=1, + min=1, + max=4096, + display_mode=io.NumberDisplay.number, + tooltip="The number of latent audio samples in the batch.", + ), + io.Vae.Input( + id="audio_vae", + display_name="Audio VAE", + tooltip="The Audio VAE model to get configuration from.", + ), + ], + outputs=[io.Latent.Output(display_name="Latent")], + ) + + @classmethod + def execute( + cls, + frames_number: int, + frame_rate: float, + batch_size: int, + audio_vae, + ) -> io.NodeOutput: + """Generate empty audio latents matching the reference pipeline structure.""" + + assert audio_vae is not None, "Audio VAE model is required" + + z_channels = audio_vae.latent_channels + audio_freq = audio_vae.first_stage_model.latent_frequency_bins + + num_audio_latents = audio_vae.first_stage_model.num_of_latents_from_frames(frames_number, frame_rate) + + audio_latents = torch.zeros( + (batch_size, z_channels, num_audio_latents, audio_freq), + device=comfy.model_management.intermediate_device(), + ) + + return io.NodeOutput( + { + "samples": audio_latents, + "type": "audio", + } + ) + + +class LTXAVTextEncoderLoader(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="LTXAVTextEncoderLoader", + display_name="Load LTXV Audio Text Encoder", + category="model/loaders", + description="Recipes:\nltxav: gemma 3 12B", + inputs=[ + io.Combo.Input( + "text_encoder", + options=folder_paths.get_filename_list("text_encoders"), + ), + io.Combo.Input( + "ckpt_name", + options=folder_paths.get_filename_list("checkpoints"), + ), + io.Combo.Input( + "device", + options=["default", "cpu"], + advanced=True, + ) + ], + outputs=[io.Clip.Output()], + ) + + @classmethod + def execute(cls, text_encoder, ckpt_name, device="default"): + clip_type = comfy.sd.CLIPType.LTXV + + clip_path1 = folder_paths.get_full_path_or_raise("text_encoders", text_encoder) + clip_path2 = folder_paths.get_full_path_or_raise("checkpoints", ckpt_name) + + model_options = {} + if device == "cpu": + model_options["load_device"] = model_options["offload_device"] = torch.device("cpu") + + clip = comfy.sd.load_clip(ckpt_paths=[clip_path1, clip_path2], embedding_directory=folder_paths.get_folder_paths("embeddings"), clip_type=clip_type, model_options=model_options) + return io.NodeOutput(clip) + + +class LTXVAudioExtension(ComfyExtension): + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + LTXVAudioVAELoader, + LTXVAudioVAEEncode, + LTXVAudioVAEDecode, + LTXVEmptyLatentAudio, + LTXAVTextEncoderLoader, + ] + + +async def comfy_entrypoint() -> ComfyExtension: + return LTXVAudioExtension() diff --git a/ComfyUI/comfy_extras/nodes_lt_upsampler.py b/ComfyUI/comfy_extras/nodes_lt_upsampler.py new file mode 100644 index 0000000000000000000000000000000000000000..7e7975495cb6a722d5aa06700c2354edd2861022 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_lt_upsampler.py @@ -0,0 +1,75 @@ +from comfy import model_management +from comfy_api.latest import ComfyExtension, IO +from typing_extensions import override +import math + + +class LTXVLatentUpsampler(IO.ComfyNode): + """ + Upsamples a video latent by a factor of 2. + """ + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="LTXVLatentUpsampler", + category="model/latent/ltxv", + is_experimental=True, + inputs=[ + IO.Latent.Input("samples"), + IO.LatentUpscaleModel.Input("upscale_model"), + IO.Vae.Input("vae"), + ], + outputs=[ + IO.Latent.Output(), + ], + ) + + @classmethod + def execute(cls, samples, upscale_model, vae) -> IO.NodeOutput: + """ + Upsample the input latent using the provided model. + + Args: + samples (dict): Input latent samples + upscale_model (LatentUpsampler): Loaded upscale model + vae: VAE model for normalization + + Returns: + tuple: Tuple containing the upsampled latent + """ + device = upscale_model.load_device + model = upscale_model.model + model_dtype = upscale_model.model_dtype() + latents = samples["samples"] + input_dtype = latents.dtype + + memory_required = math.prod(latents.shape) * 3000.0 # TODO: more accurate + model_management.load_models_gpu([upscale_model], memory_required=memory_required) + + latents = latents.to(dtype=model_dtype, device=device) + + """Upsample latents without tiling.""" + latents = vae.first_stage_model.per_channel_statistics.un_normalize(latents) + upsampled_latents = model(latents) + + upsampled_latents = vae.first_stage_model.per_channel_statistics.normalize( + upsampled_latents + ) + upsampled_latents = upsampled_latents.to(dtype=input_dtype, device=model_management.intermediate_device()) + return_dict = samples.copy() + return_dict["samples"] = upsampled_latents + return_dict.pop("noise_mask", None) + return IO.NodeOutput(return_dict) + + upsample_latent = execute # TODO: remove + + +class LTXVLatentUpsamplerExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[IO.ComfyNode]]: + return [LTXVLatentUpsampler] + + +async def comfy_entrypoint() -> LTXVLatentUpsamplerExtension: + return LTXVLatentUpsamplerExtension() diff --git a/ComfyUI/comfy_extras/nodes_lumina2.py b/ComfyUI/comfy_extras/nodes_lumina2.py new file mode 100644 index 0000000000000000000000000000000000000000..bc543c242159af8b5ac1f800492ddb9b3f7e6933 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_lumina2.py @@ -0,0 +1,128 @@ +from typing_extensions import override +import torch + +from comfy_api.latest import ComfyExtension, io + + +class RenormCFG(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="RenormCFG", + category="model/patch", + inputs=[ + io.Model.Input("model"), + io.Float.Input("cfg_trunc", default=100, min=0.0, max=100.0, step=0.01, advanced=True), + io.Float.Input("renorm_cfg", default=1.0, min=0.0, max=100.0, step=0.01, advanced=True), + ], + outputs=[ + io.Model.Output(), + ], + ) + + @classmethod + def execute(cls, model, cfg_trunc, renorm_cfg) -> io.NodeOutput: + def renorm_cfg_func(args): + cond_denoised = args["cond_denoised"] + uncond_denoised = args["uncond_denoised"] + cond_scale = args["cond_scale"] + timestep = args["timestep"] + x_orig = args["input"] + in_channels = model.model.diffusion_model.in_channels + + if timestep[0] < cfg_trunc: + cond_eps, uncond_eps = cond_denoised[:, :in_channels], uncond_denoised[:, :in_channels] + cond_rest, _ = cond_denoised[:, in_channels:], uncond_denoised[:, in_channels:] + half_eps = uncond_eps + cond_scale * (cond_eps - uncond_eps) + half_rest = cond_rest + + if float(renorm_cfg) > 0.0: + ori_pos_norm = torch.linalg.vector_norm(cond_eps + , dim=tuple(range(1, len(cond_eps.shape))), keepdim=True + ) + max_new_norm = ori_pos_norm * float(renorm_cfg) + new_pos_norm = torch.linalg.vector_norm( + half_eps, dim=tuple(range(1, len(half_eps.shape))), keepdim=True + ) + if new_pos_norm >= max_new_norm: + half_eps = half_eps * (max_new_norm / new_pos_norm) + else: + cond_eps, uncond_eps = cond_denoised[:, :in_channels], uncond_denoised[:, :in_channels] + cond_rest, _ = cond_denoised[:, in_channels:], uncond_denoised[:, in_channels:] + half_eps = cond_eps + half_rest = cond_rest + + cfg_result = torch.cat([half_eps, half_rest], dim=1) + + # cfg_result = uncond_denoised + (cond_denoised - uncond_denoised) * cond_scale + + return x_orig - cfg_result + + m = model.clone() + m.set_model_sampler_cfg_function(renorm_cfg_func) + return io.NodeOutput(m) + + +class CLIPTextEncodeLumina2(io.ComfyNode): + SYSTEM_PROMPT = { + "superior": "You are an assistant designed to generate superior images with the superior "\ + "degree of image-text alignment based on textual prompts or user prompts.", + "alignment": "You are an assistant designed to generate high-quality images with the "\ + "highest degree of image-text alignment based on textual prompts." + } + SYSTEM_PROMPT_TIP = "Lumina2 provide two types of system prompts:" \ + "Superior: You are an assistant designed to generate superior images with the superior "\ + "degree of image-text alignment based on textual prompts or user prompts. "\ + "Alignment: You are an assistant designed to generate high-quality images with the highest "\ + "degree of image-text alignment based on textual prompts." + @classmethod + def define_schema(cls): + return io.Schema( + node_id="CLIPTextEncodeLumina2", + search_aliases=["lumina prompt"], + display_name="CLIP Text Encode (Lumina 2)", + category="model/conditioning/lumina", + description="Encodes a system prompt and a user prompt using a CLIP model into an embedding " + "that can be used to guide the diffusion model towards generating specific images.", + inputs=[ + io.Combo.Input( + "system_prompt", + options=list(cls.SYSTEM_PROMPT.keys()), + tooltip=cls.SYSTEM_PROMPT_TIP, + ), + io.String.Input( + "user_prompt", + multiline=True, + dynamic_prompts=True, + tooltip="The text to be encoded.", + ), + io.Clip.Input("clip", tooltip="The CLIP model used for encoding the text."), + ], + outputs=[ + io.Conditioning.Output( + tooltip="A conditioning containing the embedded text used to guide the diffusion model.", + ), + ], + ) + + @classmethod + def execute(cls, clip, user_prompt, system_prompt) -> io.NodeOutput: + if clip is None: + raise RuntimeError("ERROR: clip input is invalid: None\n\nIf the clip is from a checkpoint loader node your checkpoint does not contain a valid clip or text encoder model.") + system_prompt = cls.SYSTEM_PROMPT[system_prompt] + prompt = f'{system_prompt} {user_prompt}' + tokens = clip.tokenize(prompt) + return io.NodeOutput(clip.encode_from_tokens_scheduled(tokens)) + + +class Lumina2Extension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + CLIPTextEncodeLumina2, + RenormCFG, + ] + + +async def comfy_entrypoint() -> Lumina2Extension: + return Lumina2Extension() diff --git a/ComfyUI/comfy_extras/nodes_mage.py b/ComfyUI/comfy_extras/nodes_mage.py new file mode 100644 index 0000000000000000000000000000000000000000..a3b0d394c29d72c2bbdecbc80b0d972a63ddcab2 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_mage.py @@ -0,0 +1,103 @@ +from typing_extensions import override + +import comfy.utils +import node_helpers +import torch +import comfy.model_management +from comfy_api.latest import ComfyExtension, io + + +class TextEncodeMageFlowEdit(io.ComfyNode): + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="TextEncodeMageFlowEdit", + category="model/conditioning/mage", + description="Encode an edit instruction with one or more reference images for Mage-Flow-Edit. Reference latents are resized to the output resolution (width/height, or the first image's size when 0). Use the latent output for sampling so the sizes always match.", + inputs=[ + io.Clip.Input("clip"), + io.String.Input("prompt", multiline=True, dynamic_prompts=True), + io.String.Input("negative_prompt", multiline=True, dynamic_prompts=True, advanced=True), + io.Vae.Input("vae", optional=True), + io.Autogrow.Input( + "images", + template=io.Autogrow.TemplateNames( + io.Image.Input("image"), + names=[f"image_{i}" for i in range(1, 17)], + min=0, + ), + tooltip="Reference image(s) to edit. All references are resized to the output resolution before encoding.", + ), + io.Int.Input("width", default=0, min=0, max=8192, step=16, tooltip="Output width. 0 = use the first reference image's size."), + io.Int.Input("height", default=0, min=0, max=8192, step=16, tooltip="Output height. 0 = use the first reference image's size."), + io.Int.Input("batch_size", default=1, min=1, max=4096), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + io.Latent.Output(display_name="latent"), + ], + ) + + @classmethod + def execute(cls, clip, prompt, negative_prompt="", vae=None, images: io.Autogrow.Type = None, width=0, height=0, batch_size=1) -> io.NodeOutput: + ref_latents = [] + images = images or {} + images = [images[name] for name in sorted(images, key=lambda n: int(n.rsplit("_", 1)[-1])) if images[name] is not None] + images_vl = [] + + # Output resolution: explicit width/height, else the primary reference's own size, floored to /16. + # Each dimension falls back independently so a 0 on one axis keeps an explicit value on the other. + if width == 0 or height == 0: + if len(images) > 0: + ref_h, ref_w = images[0].shape[1], images[0].shape[2] + else: + ref_h, ref_w = 1024, 1024 + height = height or ref_h + width = width or ref_w + width = max(16, (width // 16) * 16) + height = max(16, (height // 16) * 16) + + for image in images: + samples = image.movedim(-1, 1) + + # VL conditioning copy: cap the long edge at 384 (training preprocessing). + long_edge = max(samples.shape[3], samples.shape[2]) + if long_edge > 384: + scale_by = 384 / long_edge + s = comfy.utils.common_upscale(samples, max(1, round(samples.shape[3] * scale_by)), max(1, round(samples.shape[2] * scale_by)), "bicubic", "disabled") + images_vl.append(s.movedim(1, -1)) + else: + images_vl.append(image) + + if vae is not None: + # All references are resized to the output resolution before encoding, because Mage's RoPE aligns reference and target content by position + if samples.shape[3] != width or samples.shape[2] != height: + s = comfy.utils.common_upscale(samples, width, height, "bicubic", "disabled") + else: + s = samples + ref_latents.append(vae.encode(s.movedim(1, -1)[:, :, :, :3])) + + # Negative branch keeps the same reference images (VL tokens + ref latents), only the instruction differs. + positive = clip.encode_from_tokens_scheduled(clip.tokenize(prompt, images=images_vl)) + negative = clip.encode_from_tokens_scheduled(clip.tokenize(negative_prompt if negative_prompt else " ", images=images_vl)) + + if len(ref_latents) > 0: + positive = node_helpers.conditioning_set_values(positive, {"reference_latents": ref_latents}, append=True) + negative = node_helpers.conditioning_set_values(negative, {"reference_latents": ref_latents}, append=True) + + latent = torch.zeros([batch_size, 128, height // 16, width // 16], device=comfy.model_management.intermediate_device()) + return io.NodeOutput(positive, negative, {"samples": latent}) + + +class MageExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + TextEncodeMageFlowEdit, + ] + + +async def comfy_entrypoint() -> MageExtension: + return MageExtension() diff --git a/ComfyUI/comfy_extras/nodes_mahiro.py b/ComfyUI/comfy_extras/nodes_mahiro.py new file mode 100644 index 0000000000000000000000000000000000000000..7bd5f6652881719341f4afc55a7981a098821fcb --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_mahiro.py @@ -0,0 +1,65 @@ +from typing_extensions import override +import torch +import torch.nn.functional as F + +from comfy_api.latest import ComfyExtension, io + + +class Mahiro(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="Mahiro", + display_name="Positive-Biased Guidance", + category="experimental", + description="Modify the guidance to scale more on the 'direction' of the positive prompt rather than the difference between the negative prompt.", + inputs=[ + io.Model.Input("model"), + ], + outputs=[ + io.Model.Output(display_name="patched_model"), + ], + is_experimental=True, + search_aliases=[ + "mahiro", + "mahiro cfg", + "similarity-adaptive guidance", + "positive-biased cfg", + ], + ) + + @classmethod + def execute(cls, model) -> io.NodeOutput: + m = model.clone() + + def mahiro_normd(args): + scale: float = args["cond_scale"] + cond_p: torch.Tensor = args["cond_denoised"] + uncond_p: torch.Tensor = args["uncond_denoised"] + # naive leap + leap = cond_p * scale + # sim with uncond leap + u_leap = uncond_p * scale + cfg = args["denoised"] + merge = (leap + cfg) / 2 + normu = torch.sqrt(u_leap.abs()) * u_leap.sign() + normm = torch.sqrt(merge.abs()) * merge.sign() + sim = F.cosine_similarity(normu, normm).mean() + simsc = 2 * (sim + 1) + wm = (simsc * cfg + (4 - simsc) * leap) / 4 + return wm + + m.set_model_sampler_post_cfg_function(mahiro_normd) + return io.NodeOutput(m) + + +class MahiroExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + Mahiro, + ] + + +async def comfy_entrypoint() -> MahiroExtension: + return MahiroExtension() diff --git a/ComfyUI/comfy_extras/nodes_mask.py b/ComfyUI/comfy_extras/nodes_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..3fae7221f18dc18547a3ef4806cbc4518162093f --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_mask.py @@ -0,0 +1,457 @@ +import numpy as np +import scipy.ndimage +import torch +import comfy.utils +import comfy.model_management +import node_helpers +from typing_extensions import override +from comfy_api.latest import ComfyExtension, IO, UI + +import nodes + +def composite(destination, source, x, y, mask = None, multiplier = 8, resize_source = False): + source = source.to(destination.device) + if resize_source: + source = torch.nn.functional.interpolate(source, size=(destination.shape[-2], destination.shape[-1]), mode="bilinear") + + source = comfy.utils.repeat_to_batch_size(source, destination.shape[0]) + + x = max(-source.shape[-1] * multiplier, min(x, destination.shape[-1] * multiplier)) + y = max(-source.shape[-2] * multiplier, min(y, destination.shape[-2] * multiplier)) + + left, top = (x // multiplier, y // multiplier) + right, bottom = (left + source.shape[-1], top + source.shape[-2],) + + if mask is None: + mask = torch.ones_like(source) + else: + mask = mask.to(destination.device, copy=True) + mask = torch.nn.functional.interpolate(mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])), size=(source.shape[-2], source.shape[-1]), mode="bilinear") + mask = comfy.utils.repeat_to_batch_size(mask, source.shape[0]) + + # calculate the bounds of the source that will be overlapping the destination + # this prevents the source trying to overwrite latent pixels that are out of bounds + # of the destination + visible_width, visible_height = (destination.shape[-1] - left + min(0, x), destination.shape[-2] - top + min(0, y),) + + mask = mask[:, :, :visible_height, :visible_width] + if mask.ndim < source.ndim: + mask = mask.unsqueeze(1) + + inverse_mask = torch.ones_like(mask) - mask + + source_portion = mask * source[..., :visible_height, :visible_width] + destination_portion = inverse_mask * destination[..., top:bottom, left:right] + + destination[..., top:bottom, left:right] = source_portion + destination_portion + return destination + + +class LatentCompositeMasked(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="LatentCompositeMasked", + search_aliases=["overlay latent", "layer latent", "paste latent", "inpaint latent"], + display_name="Latent Composite Masked", + category="model/latent", + inputs=[ + IO.Latent.Input("destination"), + IO.Latent.Input("source"), + IO.Int.Input("x", default=0, min=0, max=nodes.MAX_RESOLUTION, step=8), + IO.Int.Input("y", default=0, min=0, max=nodes.MAX_RESOLUTION, step=8), + IO.Boolean.Input("resize_source", default=False), + IO.Mask.Input("mask", optional=True), + ], + outputs=[IO.Latent.Output()], + ) + + @classmethod + def execute(cls, destination, source, x, y, resize_source, mask = None) -> IO.NodeOutput: + output = destination.copy() + destination = destination["samples"].clone() + source = source["samples"] + output["samples"] = composite(destination, source, x, y, mask, 8, resize_source) + return IO.NodeOutput(output) + + composite = execute # TODO: remove + + +class ImageCompositeMasked(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="ImageCompositeMasked", + search_aliases=["overlay", "layer", "paste image", "images composition"], + display_name="Image Composite Masked", + category="image/compositing", + inputs=[ + IO.Image.Input("destination"), + IO.Image.Input("source"), + IO.Int.Input("x", default=0, min=0, max=nodes.MAX_RESOLUTION, step=1), + IO.Int.Input("y", default=0, min=0, max=nodes.MAX_RESOLUTION, step=1), + IO.Boolean.Input("resize_source", default=False), + IO.Mask.Input("mask", optional=True), + ], + outputs=[IO.Image.Output()], + ) + + @classmethod + def execute(cls, destination, source, x, y, resize_source, mask = None) -> IO.NodeOutput: + destination, source = node_helpers.image_alpha_fix(destination, source) + destination = destination.clone().movedim(-1, 1) + output = composite(destination, source.movedim(-1, 1), x, y, mask, 1, resize_source).movedim(1, -1) + return IO.NodeOutput(output) + + composite = execute # TODO: remove + + +class MaskToImage(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="MaskToImage", + search_aliases=["convert mask"], + display_name="Convert Mask to Image", + category="image/mask", + inputs=[ + IO.Mask.Input("mask"), + ], + outputs=[IO.Image.Output()], + ) + + @classmethod + def execute(cls, mask) -> IO.NodeOutput: + result = mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])).movedim(1, -1).expand(-1, -1, -1, 3) + return IO.NodeOutput(result) + + mask_to_image = execute # TODO: remove + + +class ImageToMask(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="ImageToMask", + search_aliases=["extract channel", "channel to mask"], + display_name="Convert Image to Mask", + category="image/mask", + inputs=[ + IO.Image.Input("image"), + IO.Combo.Input("channel", options=["red", "green", "blue", "alpha"]), + ], + outputs=[IO.Mask.Output()], + ) + + @classmethod + def execute(cls, image, channel) -> IO.NodeOutput: + channels = ["red", "green", "blue", "alpha"] + mask = image[:, :, :, channels.index(channel)] + return IO.NodeOutput(mask) + + image_to_mask = execute # TODO: remove + + +class ImageColorToMask(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="ImageColorToMask", + search_aliases=["color keying", "chroma key"], + display_name="Convert Image Color to Mask", + category="image/mask", + inputs=[ + IO.Image.Input("image"), + IO.Int.Input("color", default=0, min=0, max=0xFFFFFF, step=1, display_mode=IO.NumberDisplay.number), + ], + outputs=[IO.Mask.Output()], + ) + + @classmethod + def execute(cls, image, color) -> IO.NodeOutput: + temp = (torch.clamp(image, 0, 1.0) * 255.0).round().to(torch.int) + temp = torch.bitwise_left_shift(temp[:,:,:,0], 16) + torch.bitwise_left_shift(temp[:,:,:,1], 8) + temp[:,:,:,2] + mask = torch.where(temp == color, 1.0, 0).float() + return IO.NodeOutput(mask) + + image_to_mask = execute # TODO: remove + + +class SolidMask(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="SolidMask", + display_name="Create Solid Mask", + category="image/mask", + inputs=[ + IO.Float.Input("value", default=1.0, min=0.0, max=1.0, step=0.01), + IO.Int.Input("width", default=512, min=1, max=nodes.MAX_RESOLUTION, step=1), + IO.Int.Input("height", default=512, min=1, max=nodes.MAX_RESOLUTION, step=1), + ], + outputs=[IO.Mask.Output()], + ) + + @classmethod + def execute(cls, value, width, height) -> IO.NodeOutput: + out = torch.full((1, height, width), value, dtype=torch.float32, device=comfy.model_management.intermediate_device()) + return IO.NodeOutput(out) + + solid = execute # TODO: remove + + +class InvertMask(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="InvertMask", + search_aliases=["reverse mask", "flip mask"], + display_name="Invert Mask", + category="image/mask", + inputs=[ + IO.Mask.Input("mask"), + ], + outputs=[IO.Mask.Output()], + ) + + @classmethod + def execute(cls, mask) -> IO.NodeOutput: + out = 1.0 - mask + return IO.NodeOutput(out) + + invert = execute # TODO: remove + + +class CropMask(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="CropMask", + search_aliases=["cut mask", "extract mask region", "mask slice"], + display_name="Crop Mask", + category="image/mask", + inputs=[ + IO.Mask.Input("mask"), + IO.Int.Input("x", default=0, min=0, max=nodes.MAX_RESOLUTION, step=1), + IO.Int.Input("y", default=0, min=0, max=nodes.MAX_RESOLUTION, step=1), + IO.Int.Input("width", default=512, min=1, max=nodes.MAX_RESOLUTION, step=1), + IO.Int.Input("height", default=512, min=1, max=nodes.MAX_RESOLUTION, step=1), + ], + outputs=[IO.Mask.Output()], + ) + + @classmethod + def execute(cls, mask, x, y, width, height) -> IO.NodeOutput: + mask = mask.reshape((-1, mask.shape[-2], mask.shape[-1])) + out = mask[:, y:y + height, x:x + width] + return IO.NodeOutput(out) + + crop = execute # TODO: remove + + +class MaskComposite(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="MaskComposite", + search_aliases=["combine masks", "blend masks", "layer masks", "masks composition"], + display_name="Combine Masks", + category="image/mask", + inputs=[ + IO.Mask.Input("destination"), + IO.Mask.Input("source"), + IO.Int.Input("x", default=0, min=0, max=nodes.MAX_RESOLUTION, step=1), + IO.Int.Input("y", default=0, min=0, max=nodes.MAX_RESOLUTION, step=1), + IO.Combo.Input("operation", options=["multiply", "add", "subtract", "and", "or", "xor"]), + ], + outputs=[IO.Mask.Output()], + ) + + @classmethod + def execute(cls, destination, source, x, y, operation) -> IO.NodeOutput: + output = destination.reshape((-1, destination.shape[-2], destination.shape[-1])).clone() + source = source.reshape((-1, source.shape[-2], source.shape[-1])) + source = source.to(output.device) + + left, top = (x, y,) + right, bottom = (min(left + source.shape[-1], destination.shape[-1]), min(top + source.shape[-2], destination.shape[-2])) + visible_width, visible_height = (right - left, bottom - top,) + + source_portion = source[:, :visible_height, :visible_width] + destination_portion = output[:, top:bottom, left:right] + + if operation == "multiply": + output[:, top:bottom, left:right] = destination_portion * source_portion + elif operation == "add": + output[:, top:bottom, left:right] = destination_portion + source_portion + elif operation == "subtract": + output[:, top:bottom, left:right] = destination_portion - source_portion + elif operation == "and": + output[:, top:bottom, left:right] = torch.bitwise_and(destination_portion.round().bool(), source_portion.round().bool()).float() + elif operation == "or": + output[:, top:bottom, left:right] = torch.bitwise_or(destination_portion.round().bool(), source_portion.round().bool()).float() + elif operation == "xor": + output[:, top:bottom, left:right] = torch.bitwise_xor(destination_portion.round().bool(), source_portion.round().bool()).float() + + output = torch.clamp(output, 0.0, 1.0) + + return IO.NodeOutput(output) + + combine = execute # TODO: remove + + +class FeatherMask(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="FeatherMask", + search_aliases=["soft edge mask", "blur mask edges", "gradient mask edge"], + display_name="Feather Mask", + category="image/mask", + inputs=[ + IO.Mask.Input("mask"), + IO.Int.Input("left", default=0, min=0, max=nodes.MAX_RESOLUTION, step=1), + IO.Int.Input("top", default=0, min=0, max=nodes.MAX_RESOLUTION, step=1), + IO.Int.Input("right", default=0, min=0, max=nodes.MAX_RESOLUTION, step=1), + IO.Int.Input("bottom", default=0, min=0, max=nodes.MAX_RESOLUTION, step=1), + ], + outputs=[IO.Mask.Output()], + ) + + @classmethod + def execute(cls, mask, left, top, right, bottom) -> IO.NodeOutput: + output = mask.reshape((-1, mask.shape[-2], mask.shape[-1])).clone() + + left = min(left, output.shape[-1]) + right = min(right, output.shape[-1]) + top = min(top, output.shape[-2]) + bottom = min(bottom, output.shape[-2]) + + for x in range(left): + feather_rate = (x + 1.0) / left + output[:, :, x] *= feather_rate + + for x in range(right): + feather_rate = (x + 1) / right + output[:, :, -(x + 1)] *= feather_rate + + for y in range(top): + feather_rate = (y + 1) / top + output[:, y, :] *= feather_rate + + for y in range(bottom): + feather_rate = (y + 1) / bottom + output[:, -(y + 1), :] *= feather_rate + + return IO.NodeOutput(output) + + feather = execute # TODO: remove + + +class GrowMask(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="GrowMask", + search_aliases=["expand mask", "shrink mask"], + display_name="Grow Mask", + category="image/mask", + inputs=[ + IO.Mask.Input("mask"), + IO.Int.Input("expand", default=0, min=-nodes.MAX_RESOLUTION, max=nodes.MAX_RESOLUTION, step=1), + IO.Boolean.Input("tapered_corners", default=True, advanced=True), + ], + outputs=[IO.Mask.Output()], + ) + + @classmethod + def execute(cls, mask, expand, tapered_corners) -> IO.NodeOutput: + c = 0 if tapered_corners else 1 + kernel = np.array([[c, 1, c], + [1, 1, 1], + [c, 1, c]]) + mask = mask.reshape((-1, mask.shape[-2], mask.shape[-1])) + out = [] + for m in mask: + output = m.numpy() + for _ in range(abs(expand)): + if expand < 0: + output = scipy.ndimage.grey_erosion(output, footprint=kernel) + else: + output = scipy.ndimage.grey_dilation(output, footprint=kernel) + output = torch.from_numpy(output) + out.append(output) + return IO.NodeOutput(torch.stack(out, dim=0)) + + expand_mask = execute # TODO: remove + +class ThresholdMask(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="ThresholdMask", + search_aliases=["binary mask"], + display_name="Threshold Mask", + category="image/mask", + inputs=[ + IO.Mask.Input("mask"), + IO.Float.Input("value", default=0.5, min=0.0, max=1.0, step=0.01), + ], + outputs=[IO.Mask.Output()], + ) + + @classmethod + def execute(cls, mask, value) -> IO.NodeOutput: + mask = (mask > value).float() + return IO.NodeOutput(mask) + + image_to_mask = execute # TODO: remove + + +# Mask Preview - original implement from +# https://github.com/cubiq/ComfyUI_essentials/blob/9d9f4bedfc9f0321c19faf71855e228c93bd0dc9/mask.py#L81 +# upstream requested in https://github.com/Kosinkadink/rfcs/blob/main/rfcs/0000-corenodes.md#preview-nodes +class MaskPreview(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="MaskPreview", + search_aliases=["show mask", "view mask", "inspect mask", "debug mask"], + display_name="Preview Mask", + category="image/mask", + description="Preview the masks without saving them to the ComfyUI output directory.", + inputs=[ + IO.Mask.Input("mask"), + ], + hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], + is_output_node=True, + outputs=[IO.Mask.Output(display_name="mask")] + ) + + @classmethod + def execute(cls, mask, filename_prefix="ComfyUI") -> IO.NodeOutput: + return IO.NodeOutput(mask, ui=UI.PreviewMask(mask)) + + +class MaskExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[IO.ComfyNode]]: + return [ + LatentCompositeMasked, + ImageCompositeMasked, + MaskToImage, + ImageToMask, + ImageColorToMask, + SolidMask, + InvertMask, + CropMask, + MaskComposite, + FeatherMask, + GrowMask, + ThresholdMask, + MaskPreview, + ] + + +async def comfy_entrypoint() -> MaskExtension: + return MaskExtension() diff --git a/ComfyUI/comfy_extras/nodes_math.py b/ComfyUI/comfy_extras/nodes_math.py new file mode 100644 index 0000000000000000000000000000000000000000..0883c65ac7826251a0966606cb7a4cef70d5d8ee --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_math.py @@ -0,0 +1,126 @@ +"""Math expression node using simpleeval for safe evaluation. + +Provides a ComfyMathExpression node that evaluates math expressions +against dynamically-grown numeric inputs. +""" + + +import math +import string + +from simpleeval import simple_eval +from typing_extensions import override + +from comfy_api.latest import ComfyExtension, io + + +MAX_EXPONENT = 4000 + + +def _variadic_sum(*args): + """Support both sum(values) and sum(a, b, c).""" + if len(args) == 1 and hasattr(args[0], "__iter__"): + return sum(args[0]) + return sum(args) + + +def _safe_pow(base, exp): + """Wrap pow() with an exponent cap to prevent DoS via huge exponents. + + The ** operator is already guarded by simpleeval's safe_power, but + pow() as a callable bypasses that guard. + """ + if abs(exp) > MAX_EXPONENT: + raise ValueError(f"Exponent {exp} exceeds maximum allowed ({MAX_EXPONENT})") + return pow(base, exp) + + +MATH_FUNCTIONS = { + "sum": _variadic_sum, + "min": min, + "max": max, + "abs": abs, + "round": round, + "pow": _safe_pow, + "sqrt": math.sqrt, + "ceil": math.ceil, + "floor": math.floor, + "log": math.log, + "log2": math.log2, + "log10": math.log10, + "sin": math.sin, + "cos": math.cos, + "tan": math.tan, + "int": int, + "float": float, +} + + +class MathExpressionNode(io.ComfyNode): + """Evaluates a math expression against dynamically-grown inputs.""" + + @classmethod + def define_schema(cls) -> io.Schema: + autogrow = io.Autogrow.TemplateNames( + input=io.MultiType.Input("value", [io.Float, io.Int, io.Boolean]), + names=list(string.ascii_lowercase), + min=1, + ) + return io.Schema( + node_id="ComfyMathExpression", + display_name="Math Expression", + category="utilities", + search_aliases=[ + "expression", "formula", "calculate", "calculator", + "eval", "math", + ], + inputs=[ + io.String.Input("expression", default="a + b", multiline=True), + io.Autogrow.Input("values", template=autogrow), + ], + outputs=[ + io.Float.Output(display_name="FLOAT"), + io.Int.Output(display_name="INT"), + io.Boolean.Output(display_name="BOOL"), + ], + ) + + @classmethod + def execute( + cls, expression: str, values: io.Autogrow.Type + ) -> io.NodeOutput: + if not expression.strip(): + raise ValueError("Expression cannot be empty.") + + context: dict = dict(values) + context["values"] = list(values.values()) + + result = simple_eval(expression, names=context, functions=MATH_FUNCTIONS) + # bool check must come first because bool is a subclass of int in Python + if not isinstance(result, (int, float)): + raise ValueError( + f"Math Expression '{expression}' must evaluate to a numeric result, " + f"got {type(result).__name__}: {result!r}" + ) + try: + float_result = float(result) + except OverflowError: + raise ValueError( + f"Math Expression '{expression}' produced a result too large to " + f"represent as a float: {result}" + ) from None + if not math.isfinite(float_result): + raise ValueError( + f"Math Expression '{expression}' produced a non-finite result: {result}" + ) + return io.NodeOutput(float_result, int(result), bool(result)) + + +class MathExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [MathExpressionNode] + + +async def comfy_entrypoint() -> MathExtension: + return MathExtension() diff --git a/ComfyUI/comfy_extras/nodes_mediapipe.py b/ComfyUI/comfy_extras/nodes_mediapipe.py new file mode 100644 index 0000000000000000000000000000000000000000..343d88dbb9c8d8df2a019f8436fa8c896108770a --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_mediapipe.py @@ -0,0 +1,508 @@ +"""ComfyUI nodes for the pure-PyTorch MediaPipe Face Landmarker port. + +Custom IO types: + FACE_LANDMARKER — FaceLandmarkerModel wrapper (ModelPatcher inside) + FACE_LANDMARKS — {"frames": List[List[face_dict]], "image_size": (H, W), + "connection_sets": dict[str, frozenset[(int, int)]]} + face_dict: bbox_xyxy, blendshapes, landmarks_xy, + landmarks_3d, presence, score, transformation_matrix + +MediaPipeFaceLandmarker also emits the core BOUNDING_BOX type — pair with DrawBBoxes. +""" + + +import numpy as np +import torch +from PIL import Image, ImageColor, ImageDraw +from tqdm.auto import tqdm +from typing_extensions import override + +import comfy.model_management +import comfy.model_patcher +import comfy.utils +import folder_paths +from comfy_api.latest import ComfyExtension, io + +from comfy_extras.mediapipe.face_landmarker import FaceLandmarker +from comfy_extras.mediapipe.face_geometry import transformation_matrix_from_detection + + +FaceDetectionType = io.Custom("FACE_DETECTION_MODEL") +FaceLandmarksType = io.Custom("FACE_LANDMARKS") + +_CANONICAL_KEYS = ("canonical_vertices", "procrustes_indices", "procrustes_weights") +_CONTOUR_PARTS = ("face_oval", "left_eye", "right_eye", "left_eyebrow", "right_eyebrow", "lips") + + +class FaceLandmarkerModel: + """Loaded FaceLandmarker variants + ModelPatcher per variant. + + Safetensors layout: `detector_short.*` / `detector_full.*` plus shared + `mesh.*`, `blendshapes.*`, `canonical_*`, and `topology.*`. + PReLU forces plain-nn / fp32 (manual_cast strands buffers across devices). + """ + + def __init__(self, state_dict: dict): + self.load_device = comfy.model_management.text_encoder_device() + offload_device = comfy.model_management.text_encoder_offload_device() + self.dtype = torch.float32 + + # FACEMESH_* connection sets, embedded as int32 (N, 2) under topology.*. + base: dict[str, frozenset] = {} + for k in [k for k in state_dict if k.startswith("topology.")]: + base[k[len("topology."):]] = frozenset(map(tuple, state_dict.pop(k).tolist())) + base["contours"] = frozenset().union(*(base[p] for p in _CONTOUR_PARTS)) + base["all"] = base["contours"] | base["irises"] | base["nose"] + + self.connection_sets: dict[str, frozenset] = base + self.canonical_data: dict[str, np.ndarray] = {k: state_dict.pop(k).numpy() for k in _CANONICAL_KEYS} + + shared = {k: v for k, v in state_dict.items() if k.startswith(("mesh.", "blendshapes."))} + + self.models: dict[str, FaceLandmarker] = {} + self.patchers: dict[str, comfy.model_patcher.ModelPatcher] = {} + for variant in ("short", "full"): + prefix = f"detector_{variant}." + sub = dict(shared) + sub.update({f"detector.{k[len(prefix):]}": v for k, v in state_dict.items() if k.startswith(prefix)}) + fl = FaceLandmarker(device=offload_device, dtype=self.dtype, operations=None, detector_variant=variant).eval() + fl.load_state_dict(sub, strict=False) + + self.models[variant] = fl + self.patchers[variant] = comfy.model_patcher.CoreModelPatcher( + fl, load_device=self.load_device, offload_device=offload_device, + size=comfy.model_management.module_size(fl), + ) + + def detect_batch(self, images, num_faces: int, score_thresh: float, variant: str): + comfy.model_management.load_model_gpu(self.patchers[variant]) + return self.models[variant].detect_batch(images, num_faces=num_faces, score_thresh=score_thresh) + + +def _image_to_uint8(image: torch.Tensor) -> np.ndarray: + return image[..., :3].mul(255.0).add_(0.5).clamp_(0, 255).to(torch.uint8).cpu().numpy() + + +def _parse_color(color: str) -> tuple[int, int, int]: + try: + return ImageColor.getrgb(color)[:3] + except ValueError: + return (0, 255, 0) + + +def _copy_face(face: dict) -> dict: + """Shallow copy of a face_dict with array-fields cloned so callers can mutate.""" + return { + "bbox_xyxy": face["bbox_xyxy"].copy(), + "blendshapes": dict(face["blendshapes"]), + "landmarks_xy": face["landmarks_xy"].copy(), + "landmarks_3d": face["landmarks_3d"].copy(), + "presence": face["presence"], + "score": face["score"], + } + + +def _lerp_face(a: dict, b: dict, t: float) -> dict: + return { + "bbox_xyxy": (1 - t) * a["bbox_xyxy"] + t * b["bbox_xyxy"], + "blendshapes": {k: (1 - t) * a["blendshapes"][k] + t * b["blendshapes"][k] for k in a["blendshapes"]}, + "landmarks_xy": (1 - t) * a["landmarks_xy"] + t * b["landmarks_xy"], + "landmarks_3d": (1 - t) * a["landmarks_3d"] + t * b["landmarks_3d"], + "presence": (1 - t) * a["presence"] + t * b["presence"], + "score": (1 - t) * a["score"] + t * b["score"], + } + + +def _match_faces(a: list[dict], b: list[dict]) -> list[tuple[int, int]]: + """Greedy nearest-neighbour pairing of faces between two frames by bbox + centre distance. Unmatched (when counts differ) are dropped.""" + if not a or not b: + return [] + centers_a = np.array([(0.5 * (f["bbox_xyxy"][0] + f["bbox_xyxy"][2]), + 0.5 * (f["bbox_xyxy"][1] + f["bbox_xyxy"][3])) for f in a]) + centers_b = np.array([(0.5 * (f["bbox_xyxy"][0] + f["bbox_xyxy"][2]), + 0.5 * (f["bbox_xyxy"][1] + f["bbox_xyxy"][3])) for f in b]) + dists = np.linalg.norm(centers_a[:, None] - centers_b[None], axis=-1) + pairs: list[tuple[int, int]] = [] + used_a: set[int] = set() + used_b: set[int] = set() + candidates = sorted((dists[ia, ib], ia, ib) for ia in range(len(a)) for ib in range(len(b))) + for _, ia, ib in candidates: + if ia in used_a or ib in used_b: + continue + pairs.append((ia, ib)) + used_a.add(ia) + used_b.add(ib) + return pairs + + +def _fill_missing_frames(frames: list[list[dict]], mode: str) -> None: + """In-place fill empty frame slots from neighbouring detections. Multi-face + aware: pairs faces across bracketing frames by greedy bbox-centre NN. + When counts differ, unmatched faces are dropped from the synthesised frame.""" + if mode == "empty": + return + valid = [i for i, fr in enumerate(frames) if fr] + if not valid: + return # nothing to fill from + if mode == "previous": + last: list[dict] = [] + for i, fr in enumerate(frames): + if fr: + last = fr + elif last: + frames[i] = [_copy_face(f) for f in last] + return + # interpolate: lerp between bracketing valid frames; clamp at ends. + for i in range(len(frames)): + if frames[i]: + continue + prev_i = max((v for v in valid if v < i), default=None) + next_i = min((v for v in valid if v > i), default=None) + if prev_i is None: + frames[i] = [_copy_face(f) for f in frames[next_i]] + elif next_i is None: + frames[i] = [_copy_face(f) for f in frames[prev_i]] + else: + t = (i - prev_i) / (next_i - prev_i) + pairs = _match_faces(frames[prev_i], frames[next_i]) + frames[i] = [_lerp_face(frames[prev_i][a], frames[next_i][b], t) for a, b in pairs] + + +def _ordered_rings(edges: frozenset[tuple[int, int]]) -> list[list[int]]: + """Walk an unordered edge set into one or more closed-loop vertex rings + (handles multi-loop sets like FACEMESH_LIPS: outer + inner).""" + adj: dict[int, set[int]] = {} + for a, b in edges: + adj.setdefault(a, set()).add(b) + adj.setdefault(b, set()).add(a) + visited: set[int] = set() + rings: list[list[int]] = [] + for start in adj: + if start in visited: + continue + ring = [start] + visited.add(start) + prev, cur = -1, start + while True: + nxt = next((v for v in adj[cur] if v != prev), None) + if nxt is None or nxt == start: + break + ring.append(nxt) + visited.add(nxt) + prev, cur = cur, nxt + rings.append(ring) + return rings + + +class LoadMediaPipeFaceLandmarker(io.ComfyNode): + """Load MediaPipe Face Landmarker v2 weights. Contains both detector variants + (short / full), shared mesh, blendshapes, and canonical geometry.""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LoadMediaPipeFaceLandmarker", + search_aliases=["face", "facial", "mediapipe", "face landmark", "face mesh", "blazeface", "face detection"], + display_name="Load Face Detection Model (MediaPipe)", + category="model/loaders", + inputs=[ + io.Combo.Input("model_name", options=folder_paths.get_filename_list("detection"), + tooltip="Face detection model from models/detection/."), + ], + outputs=[FaceDetectionType.Output()], + ) + + @classmethod + def execute(cls, model_name) -> io.NodeOutput: + sd = comfy.utils.load_torch_file(folder_paths.get_full_path_or_raise("detection", model_name), safe_load=True) + wrapper = FaceLandmarkerModel(sd) + return io.NodeOutput(wrapper) + + +# Per-frame fallback modes for detection failures in a batch. +_FALLBACK_MODES = ("empty", "previous", "interpolate") + + +class MediaPipeFaceLandmarker(io.ComfyNode): + """BlazeFace → FaceMesh v2 → ARKit-52 blendshapes, batched across the + input. Also emits a BOUNDING_BOX list (landmark-extent bbox per face) — + pair with DrawBBoxes for detector-only viz or MediaPipeFaceMeshVisualize + for the mesh overlay.""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="MediaPipeFaceLandmarker", + search_aliases=["face", "facial", "mediapipe", "face landmark", "face mesh", "blazeface", "face detection"], + display_name="Detect Face Landmarks (MediaPipe)", + category="image/detection", + description="Detects facial landmarks using MediaPipe model.", + inputs=[ + FaceDetectionType.Input("face_detection_model"), + io.Image.Input("image"), + io.Combo.Input("detector_variant", options=["short", "full", "both"], default="short", + tooltip="Face detector range. 'short' is tuned for close-up faces " + "(within ~2 m of the camera); 'full' covers farther / smaller " + "faces (up to ~5 m) but is slower. 'both' runs both detectors and " + "keeps whichever found more faces per frame (~2× detection cost)."), + io.Int.Input("num_faces", default=1, min=0, max=16, step=1, + tooltip="Maximum faces to return per frame. 0 = no cap (return all detected)."), + io.Float.Input("min_confidence", default=0.5, min=0.0, max=1.0, step=0.01, advanced=True, + tooltip="BlazeFace score threshold. Lower to catch small/occluded faces."), + io.Combo.Input("missing_frame_fallback", options=list(_FALLBACK_MODES), default="empty", advanced=True, + tooltip="Per-frame behaviour when detection fails in a batch. " + "'empty' leaves the frame faceless. 'previous' copies the most recent successful " + "detection. 'interpolate' lerps landmarks/bbox/blendshapes between bracketing " + "successful frames. Multi-face: pairs faces across frames by greedy bbox-centre NN."), + ], + outputs=[ + FaceLandmarksType.Output(display_name="face_landmarks"), + io.BoundingBox.Output("bboxes"), + ], + ) + + @classmethod + def execute(cls, face_detection_model, image, detector_variant, num_faces, min_confidence, + missing_frame_fallback) -> io.NodeOutput: + canonical = face_detection_model.canonical_data + img_np = _image_to_uint8(image) + B, H, W = img_np.shape[:3] + chunk = 16 + is_both = detector_variant == "both" + total_work = 2 * B if is_both else B + pbar = comfy.utils.ProgressBar(total_work) + + def _run(variant: str) -> list[list[dict]]: + res: list[list[dict]] = [] + with tqdm(total=B, desc=f"MediaPipe Face Landmarker ({variant})") as tq: + for i in range(0, B, chunk): + end = min(i + chunk, B) + res.extend(face_detection_model.detect_batch( + [img_np[bi] for bi in range(i, end)], + num_faces=int(num_faces), + score_thresh=float(min_confidence), + variant=variant, + )) + pbar.update_absolute(min(pbar.current + (end - i), total_work)) + tq.update(end - i) + return res + + if is_both: + short_res = _run("short") + full_res = _run("full") + # Per-frame keep whichever found more faces (tie → short). + frames: list[list[dict]] = [ + short_res[bi] if len(short_res[bi]) >= len(full_res[bi]) else full_res[bi] + for bi in range(B) + ] + else: + frames = _run(detector_variant) + _fill_missing_frames(frames, missing_frame_fallback) + bboxes = [] + for per_frame in frames: + per_bb = [] + for f in per_frame: + f["transformation_matrix"] = transformation_matrix_from_detection(f, W, H, canonical) + x1, y1, x2, y2 = (float(v) for v in f["bbox_xyxy"]) + per_bb.append({"x": x1, "y": y1, "width": x2 - x1, "height": y2 - y1, "label": "face", "score": float(f["score"])}) + bboxes.append(per_bb) + return io.NodeOutput({"frames": frames, "image_size": (H, W), + "connection_sets": face_detection_model.connection_sets}, bboxes) + + +# Topology keys unioned by the 'all' connections preset (contour parts + irises + nose). +_ALL_CONNECTION_PARTS: tuple[str, ...] = (*_CONTOUR_PARTS, "irises", "nose") +_CUSTOM_FEATURES: tuple[tuple[str, bool], ...] = ( + ("face_oval", True), + ("lips", True), + ("left_eye", True), + ("right_eye", True), + ("left_eyebrow", True), + ("right_eyebrow", True), + ("irises", True), + ("nose", True), + ("tesselation", False), +) + + +class MediaPipeFaceMeshVisualize(io.ComfyNode): + """Draw a FACEMESH_* subset over an image. Topology travels with the + FACE_LANDMARKS payload (set at detection time).""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="MediaPipeFaceMeshVisualize", + search_aliases=["face", "facial", "mediapipe", "face landmark", "face mesh", "blazeface", "face detection", "visualize"], + display_name="Visualize Face Landmarks (MediaPipe)", + category="image/detection", + description="Draws face landmarks mesh on the input image.", + inputs=[ + FaceLandmarksType.Input("face_landmarks"), + io.Image.Input("image", optional=True, tooltip="If not connected, a black canvas will be used."), + io.DynamicCombo.Input( + "connections", + tooltip="'all' = oval+eyes+brows+lips+irises+nose. 'fill' = solid face_oval polygon (silhouette mask). 'custom' = toggle each feature individually (including 'tesselation', the full 2547-edge wireframe).", + options=[ + io.DynamicCombo.Option("all", []), + io.DynamicCombo.Option("fill", []), + io.DynamicCombo.Option("custom", [ + io.Boolean.Input(feat, default=default, + tooltip=f"Draw the '{feat}' connection set.") + for feat, default in _CUSTOM_FEATURES + ]), + ], + ), + io.Color.Input("color", default="#00ff00"), + io.Int.Input("thickness", default=1, min=0, max=8, step=1, + tooltip="Edge line thickness in pixels. 0 disables edge drawing."), + io.Int.Input("point_size", default=2, min=0, max=16, step=1, + tooltip="Landmark dot radius in pixels. 0 disables point drawing."), + ], + outputs=[io.Image.Output()], + ) + + @classmethod + def execute(cls, face_landmarks, connections, color, thickness, point_size, image=None) -> io.NodeOutput: + sets = face_landmarks["connection_sets"] + sel = connections["connections"] + fill_rings: list[list[int]] | None = None + if sel == "fill": + fill_rings = _ordered_rings(sets["face_oval"]) + edges = frozenset() + elif sel == "custom": + parts = [feat for feat, _ in _CUSTOM_FEATURES if connections.get(feat, False)] + edges = frozenset().union(*(sets[p] for p in parts)) + else: # "all" + edges = frozenset().union(*(sets[p] for p in _ALL_CONNECTION_PARTS)) + rgb, thick, psize = _parse_color(color), int(thickness), int(point_size) + frames = face_landmarks["frames"] + if image is None: + H, W = face_landmarks["image_size"] + img_np = np.zeros((len(frames), H, W, 3), dtype=np.uint8) + else: + img_np = _image_to_uint8(image) + B = img_np.shape[0] + n_frames = len(frames) + pbar = comfy.utils.ProgressBar(B) + out = np.empty_like(img_np) + for bi in range(B): + faces = frames[bi] if bi < n_frames else [] + out[bi] = _draw_mesh(img_np[bi], faces, edges, rgb, thick, psize, fill_rings) + pbar.update_absolute(bi + 1) + return io.NodeOutput(torch.from_numpy(out).to( + device=comfy.model_management.intermediate_device(), + dtype=comfy.model_management.intermediate_dtype(), + ).div_(255.0)) + + +def _draw_mesh(image_rgb: np.ndarray, faces: list, edges, + rgb: tuple[int, int, int], thickness: int, + point_size: int, fill_rings: list[list[int]] | None = None) -> np.ndarray: + draw_edges = thickness > 0 and edges + if not faces or (fill_rings is None and not draw_edges and point_size <= 0): + return image_rgb.copy() + pil = Image.fromarray(image_rgb) + draw = ImageDraw.Draw(pil) + r = point_size * 0.5 + if fill_rings is not None: + for f in faces: + lmks = f["landmarks_xy"] + for ring in fill_rings: + draw.polygon([(float(lmks[i, 0]), float(lmks[i, 1])) for i in ring], fill=rgb) + return np.asarray(pil) + for f in faces: + lmks = f["landmarks_xy"] + n = lmks.shape[0] + if draw_edges: + for a, b in edges: + if a < n and b < n: + draw.line([(float(lmks[a, 0]), float(lmks[a, 1])), + (float(lmks[b, 0]), float(lmks[b, 1]))], fill=rgb, width=thickness) + if point_size == 1: + draw.point(lmks.flatten().tolist(), fill=rgb) + elif point_size > 1: + for x, y in lmks: + draw.ellipse((float(x) - r, float(y) - r, float(x) + r, float(y) + r), fill=rgb) + return np.asarray(pil) + + +# Mask region presets — closed-loop topologies only. +_MASK_REGIONS: tuple[str, ...] = ("face_oval", "lips", "left_eye", "right_eye", "irises") +_MASK_CUSTOM_FEATURES: tuple[tuple[str, bool], ...] = ( + ("face_oval", True), + ("lips", False), + ("left_eye", False), + ("right_eye", False), + ("irises", False), +) + + +class MediaPipeFaceMask(io.ComfyNode): + """Binary mask from face landmarks, filled polygon per face. One mask per + frame in the batch; faces in the same frame composite (union).""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="MediaPipeFaceMask", + search_aliases=["face", "facial", "mediapipe", "face mask", "blazeface", "face detection", "visualize"], + display_name="Draw Face Mask (MediaPipe)", + category="image/detection", + description="Draws a mask from face landmarks.", + inputs=[ + FaceLandmarksType.Input("face_landmarks"), + io.DynamicCombo.Input( + "regions", + tooltip="'all' = union of face_oval+lips+eyes+irises (which collapses to face_oval since it encloses the rest). 'custom' = toggle each region individually for combos like lips+eyes.", + options=[ + io.DynamicCombo.Option("all", []), + io.DynamicCombo.Option("custom", [ + io.Boolean.Input(reg, default=default, + tooltip=f"Include the '{reg}' region in the mask.") + for reg, default in _MASK_CUSTOM_FEATURES + ]), + ], + ), + ], + outputs=[io.Mask.Output()], + ) + + @classmethod + def execute(cls, face_landmarks, regions) -> io.NodeOutput: + sets = face_landmarks["connection_sets"] + sel = regions["regions"] + if sel == "custom": + picked = [reg for reg, _ in _MASK_CUSTOM_FEATURES if regions.get(reg, False)] + else: + picked = list(_MASK_REGIONS) + rings = [r for reg in picked for r in _ordered_rings(sets[reg])] + frames = face_landmarks["frames"] + H, W = face_landmarks["image_size"] + masks = np.zeros((len(frames), H, W), dtype=np.uint8) + pbar = comfy.utils.ProgressBar(len(frames)) + for bi, per_frame in enumerate(frames): + if per_frame: + pil = Image.new("L", (W, H), 0) + draw = ImageDraw.Draw(pil) + for f in per_frame: + lmks = f["landmarks_xy"] + for ring in rings: + draw.polygon([(float(lmks[i, 0]), float(lmks[i, 1])) for i in ring], fill=255) + masks[bi] = np.asarray(pil) + pbar.update_absolute(bi + 1) + return io.NodeOutput(torch.from_numpy(masks).to( + device=comfy.model_management.intermediate_device(), + dtype=comfy.model_management.intermediate_dtype(), + ).div_(255.0)) + + +class MediaPipeFaceExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [LoadMediaPipeFaceLandmarker, MediaPipeFaceLandmarker, MediaPipeFaceMeshVisualize, MediaPipeFaceMask] + + +async def comfy_entrypoint() -> MediaPipeFaceExtension: + return MediaPipeFaceExtension() diff --git a/ComfyUI/comfy_extras/nodes_minimax_h3.py b/ComfyUI/comfy_extras/nodes_minimax_h3.py new file mode 100644 index 0000000000000000000000000000000000000000..22bc91cd4570a071c664ec4848a5be748909e1e7 --- /dev/null +++ b/ComfyUI/comfy_extras/nodes_minimax_h3.py @@ -0,0 +1,337 @@ +"""MiniMax H3 nodes: AV latent creation and task conditioning (t2va / fl2va / ref2va). + +The H3 packed-DiT consumes, via conditioning: +- Qwen3-VL-32B hidden states with per-token modality tags (from the minimax CLIP) +- keyframe / reference condition latents, re-injected every step (never denoised) + +Latents are NestedTensor pairs (video [B,24,T,H/16,W/16], audio [B,32,2,T40]); +sampling runs on the flat pack with any stock sampler (the model handles the +audio stream's shifted schedule internally). +""" + +import math + +import torch +import torchaudio + +import nodes +import comfy.model_management +import comfy.model_sampling +import comfy.nested_tensor +import comfy.utils +import node_helpers +from comfy_api.latest import ComfyExtension, io + +CANVAS_MULTIPLE = 32 +BASE_SHORT_EDGE = 768 +MAX_PIXELS = 768 * 1344 +REF_IMAGE_SHORT_EDGE = 2048 +FPS = 24 +AUDIO_LATENT_FPS = 40 + + +def align_frame_count(n): + while n % 17 != 5: + n += 1 + return n + + +def video_latent_t(frame_count): + return 2 if frame_count <= 5 else ((frame_count - 5) // 17) * 5 + 2 + + +def temporal_shape(length): + frame_count = align_frame_count(max(5, length)) + duration = frame_count / FPS + return frame_count, video_latent_t(frame_count), round(duration * AUDIO_LATENT_FPS) + + +def adapt_canvas(width, height): + """768-short-edge canvas with 768*1344 area cap, per-axis round to 32.""" + ratio = width / height + if ratio >= 1.0: + nom_w, nom_h = BASE_SHORT_EDGE * ratio, BASE_SHORT_EDGE + else: + nom_w, nom_h = BASE_SHORT_EDGE, BASE_SHORT_EDGE / ratio + if nom_w * nom_h > MAX_PIXELS: + s = math.sqrt(MAX_PIXELS / (nom_w * nom_h)) + nom_w, nom_h = nom_w * s, nom_h * s + return (max(CANVAS_MULTIPLE, round(nom_w / CANVAS_MULTIPLE) * CANVAS_MULTIPLE), + max(CANVAS_MULTIPLE, round(nom_h / CANVAS_MULTIPLE) * CANVAS_MULTIPLE)) + + +def _resize(image, width, height, crop): + # image [B, H, W, C] -> [B, height, width, 3] + samples = image[..., :3].movedim(-1, 1) + samples = comfy.utils.common_upscale(samples, width, height, "lanczos", crop) + return samples.movedim(1, -1) + + +def _empty_av_latent(width, height, length, batch_size=1): + frame_count, latent_t, audio_t = temporal_shape(length) + video = torch.zeros([batch_size, 24, latent_t, height // 16, width // 16], + device=comfy.model_management.intermediate_device()) + audio = torch.zeros([batch_size, 32, 2, audio_t], + device=comfy.model_management.intermediate_device()) + return {"samples": comfy.nested_tensor.NestedTensor((video, audio))}, frame_count + + +class EmptyMiniMaxH3LatentAV(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="EmptyMiniMaxH3LatentAV", + display_name="Empty MiniMax H3 AV Latent", + category="model/latent/minimax", + description="Joint video+audio latent for MiniMax H3. Duration snaps to the model's 17k+5 frame grid at 24 fps.", + inputs=[ + io.Int.Input("width", default=1344, min=32, max=nodes.MAX_RESOLUTION, step=32), + io.Int.Input("height", default=768, min=32, max=nodes.MAX_RESOLUTION, step=32), + io.Int.Input("length", default=124, min=5, max=3600, step=17, tooltip="Frame count at 24 fps, snapped up to the model's 17k+5 grid (124 = ~5s; trained range is ~124-362, longer is untested)"), + ], + outputs=[io.Latent.Output()], + ) + + @classmethod + def execute(cls, width, height, length) -> io.NodeOutput: + latent, _ = _empty_av_latent(width, height, length) + return io.NodeOutput(latent) + + +class MiniMaxH3ImageToVideo(io.ComfyNode): + """t2va and fl2va: prompt (+ optional first/last keyframes) -> conditioning + AV latent.""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="MiniMaxH3ImageToVideo", + display_name="MiniMax H3 Image to Video", + category="model/conditioning/minimax", + inputs=[ + io.Clip.Input("clip"), + io.Vae.Input("vae"), + io.String.Input("prompt", multiline=True, dynamic_prompts=True), + io.Int.Input("width", default=1344, min=32, max=nodes.MAX_RESOLUTION, step=32), + io.Int.Input("height", default=768, min=32, max=nodes.MAX_RESOLUTION, step=32), + io.Int.Input("length", default=124, min=5, max=3600, step=17, tooltip="Frame count at 24 fps, snapped up to the model's 17k+5 grid (124 = ~5s; trained range is ~124-362, longer is untested)"), + io.Image.Input("first_frame", optional=True), + io.Image.Input("last_frame", optional=True), + ], + outputs=[io.Conditioning.Output(display_name="positive"), io.Latent.Output()], + ) + + @classmethod + def execute(cls, clip, vae, prompt, width, height, length, + first_frame=None, last_frame=None) -> io.NodeOutput: + latent, frame_count = _empty_av_latent(width, height, length) + + images = [] + keyframes = [] + if first_frame is not None: + # geometry anchor: plain stretch to canvas + img = _resize(first_frame[:1], width, height, "disabled") + images.append(img) + keyframes.append({"resolved_frame_index": 0, "image": img}) + if last_frame is not None: + # follower: aspect-preserving cover-crop + img = _resize(last_frame[:1], width, height, "center") + images.append(img) + keyframes.append({"resolved_frame_index": frame_count - 1, "image": img}) + + tokens = clip.tokenize(prompt, images=images) + cond = clip.encode_from_tokens_scheduled(tokens) + + if keyframes: + for kf in keyframes: + kf["latent"] = vae.encode(kf.pop("image")) + cond = node_helpers.conditioning_set_values(cond, { + "minimax_keyframes": keyframes, + "minimax_frame_count": frame_count, + }) + return io.NodeOutput(cond, latent) + + +class MiniMaxH3ReferenceToVideo(io.ComfyNode): + """ref2va: prompt + reference images / videos / audio -> conditioning + AV latent. + + References enter the presentation in fixed order: images, then videos (each + soundtrack's

🇬🇧 English

+ +> A highly professional, multi-functional ComfyUI model browser plugin. It integrates advanced model management, workflow healing, an intelligent drafting notebook, and a native output gallery. + +### 🌟 Core Capabilities + +| 🚀 Feature | 💡 Description | +| :--- | :--- | +| **Comprehensive Model Management** | Automatically extract Base Model architectures, cover images, trigger words, and author info by scanning model hashes. Customize model names, notes, and covers. Safely delete unwanted models or quickly replace them visually with cover icons. | +| **Workflow Node Repair (🩺)** | As long as a workflow/image was exported with this plugin active, the **Model Doctor (🩺)** can auto-detect missing nodes and replace them with correct local paths. Stubborn models? Use "Deep Hash Scan" or get Civitai links. | +| **Smart Notebook (📑)** | Leveraging scanned local architectures, it offers rapid matching between Checkpoints, UNet, and compatible LoRAs. Features a built-in translation tool, auto-chunks prompts into tags, and supports 1-click canvas deployment. | +| **History Gallery (🖼️)** | Natively reads your local `output` folder. Supports 1-click viewing, deleting, and mouse-wheel zooming. Drag and drop any generated image directly onto the canvas to instantly load its embedded workflow! | +| **Additional Features** | Click the top-right `➕` of any model card to deploy it to the canvas. The UI supports side-docking to leave space for your canvas operations. | + +### 📖 Step-by-Step Operating Guide + +#### 1. Initialization (Scan Wizard) 🔄 +* **Location**: The Scan icon (**🔄**) at the bottom left of the plugin interface. +* **Operation**: This is the most important prerequisite. Open the Scan Wizard, select your configuration, and execute the scan. Ensure your network is connected. +* **Result**: This activates all core features of the plugin by building your local model database. + +#### 2. Workflow Repair (Model Doctor) 🩺 +* **Location**: The second Stethoscope icon (**🩺**) from the bottom left. +* **Operation**: When you import someone else's workflow and the nodes turn red due to missing paths, click the Model Doctor. It will intelligently identify all broken nodes and replace them with your correct local paths. + +#### 3. Visual Swapping (Node Assistant) 🤖 +* **Location**: The third Robot icon (**🤖**) from the bottom left. +* **Operation**: After opening, select a model node on your ComfyUI canvas. The redesigned action card can visually replace its current model or insert a LoRA before/after a compatible MODEL + CLIP chain. Picker cards show both model category and base-model badges. When choosing a LoRA, the connected main model's type is selected automatically when metadata is available; the filter can always be switched back to all LoRAs. Folder browsing, full-path search, sorting, and safe handling of ambiguous branches remain available. + +#### 4. Settings Panel ⚙️ +* **Location**: The Gear icon (**⚙️**) at the bottom left. +* **Operation**: Here you can adjust the UI language and main UI font size. Open **Model Settings** to choose always-play or hover-play video covers and optimized card thumbnails or original covers. Detail pages continue to use the original cover. + +#### 5. Top Navigation & Smart Notebook 📑 +**Location**: The top tabs: **Models (📦)**, **Gallery (🖼️)**, **Notebook (📑)**, and **Dock Side (◧)**. + +**Notebook Operation**: +1. Click the **Notebook (📑)** tab, then click the **➕ (New)** button and confirm. +2. Select a Base Model (e.g., SD1.5). You can then select compatible Main Models and LoRAs. +3. Click **📝 Edit Raw / Paste** to paste your prompts. Select the language at the top-left for bilingual translation. +4. After confirming, the plugin will automatically chunk your prompt into tags. You can find/replace or directly edit tags. +5. Click **🚀 Send to Canvas** to automatically wire and deploy the Checkpoint, LoRA, and CLIP Text Encode directly to your canvas! + +--- + +### 📦 Installation + +Open your terminal, go to your ComfyUI `custom_nodes` folder, and run: +```bash +cd custom_nodes +git clone https://github.com/DemonGatanjieu/Anomalous_Model_Browser.git +``` +> **Note**: Restart ComfyUI after cloning. Alternatively, search for `Anomalous Model Browser` inside the ComfyUI Manager and click Install! + +--- + +

🇨🇳 中文

+ +> 一个高度专业、多功能的 ComfyUI 模型浏览器插件。集成了模型管理、工作流自愈修复、智能草稿本与原生图库管理。 + +### 🌟 核心功能特性 + +| 🚀 功能板块 | 💡 详细说明 | +| :--- | :--- | +| **全方位模型管理** | 通过扫描哈希值,自动获取模型的基础架构、封面、提示词、作者简介等信息。支持自定义名称、备注和封面。点开模型即刻呈现封面图标,支持看图一键替换模型,彻底告别复杂的模型路径! | +| **节点智能修复 (🩺)** | 凡是由本插件导出的工作流和图片,在载入他人工作流时出现节点爆红,只需点开**模型医生 (🩺)**,即可自动识别正确的模型路径并一键替换。内置深度扫描与 C 站直达链接。 | +| **智能笔记本 (📑)** | 基于本地基础架构,提供快速的架构匹配功能,极速选择兼容的主模型与 LoRA。内置强大的翻译功能,自动将提示词分块打上标签,实现双语对照,并支持一键发布组装到工作流画布中。 | +| **原生图库管理 (🖼️)** | 无缝读取本地的 `output` 文件夹。支持鼠标滚轮缩放、一键查看和安全删除。更绝的是,您可以**直接将图片拖动到画布上,瞬间原地加载内嵌的工作流!** | +| **其它便捷功能** | 点击模型卡片右上角的 `➕` 号,快捷将节点发布到画布。支持侧边栏停靠 (Dock),将界面吸附在左侧,为您的画布留出充足的操作空间。 | + +### 📖 标准操作指南 + +为了让插件发挥最大效能,请按照以下流程进行操作: + +#### 1. 前置准备 (扫描向导) 🔄 +* **具体位置**:界面左侧底部的 **扫描图标 (🔄)**。 +* **操作步骤**:这是最重要的一步!请先打开扫描向导,根据需求选择扫描配置并执行扫描(切记保持网络畅通)。扫描完成后,您的本地数据库建立完毕,插件的各项核心功能全面激活。 + +#### 2. 拯救爆红 (模型医生) 🩺 +* **具体位置**:左侧底部第二个 **听诊器按钮 (🩺)**。 +* **操作步骤**:当导入别人使用该插件输出的工作流或者图片时,若发现模型路径爆红报错,点击模型医生,它就能智能识别报错节点,并一键实现正确路径的替换。 + +#### 3. 选中交互 (节点助手) 🤖 +* **具体位置**:左侧底部第三个 **机器人按钮 (🤖)**。 +* **操作步骤**:打开后,在画布上选择模型节点。重新设计的操作卡可以替换当前模型,也可在兼容的 MODEL + CLIP 链前方或后方插入 LoRA。选择器卡片会标注模型类别和基础模型类型;选择 LoRA 时,如元数据可用,会自动按照已连接主模型的类型筛选,也可以随时切回全部 LoRA。文件夹浏览、完整路径搜索、排序和多分支安全保护仍然保留。 + +#### 4. 个性化配置 (设置面板) ⚙️ +* **具体位置**:左侧底部的 **齿轮按钮 (⚙️)**。 +* **操作步骤**:在此可以调节语言和主页面字体大小。进入 **模型设置** 后,可选择视频封面始终播放或悬停播放,也可选择卡片使用流畅缩略图或原始封面;模型详情页始终使用原始封面。 + +#### 5. 顶部导航与笔记本 (Notebook) 实战 📑 +**具体位置**:右侧顶部的按钮分别为 **模型 (📦)**、**图库 (🖼️)**、**笔记本 (📑)**、**停靠侧边栏 (◧)**。 + +**笔记本操作步骤**: +1. 点击**笔记本 (📑)** 按钮,点击新建 **➕** 号,确认后进入草稿本。 +2. 首先选择基础模型(例如 SD1.5),系统会过滤出兼容的主模型与对应的 LoRA 模型供你选择。 +3. 点击下方的 **📝 纯文本/粘贴** 复制提示词进去,选择左上角的语言即可实现双语对照翻译。 +4. 确认后,插件会自动生成标签 (Tags),支持查找替换,也支持直接双击修改单个标签。 +5. 最后,点击下方的 **🚀 发送到画布**,系统会将打包好的主模型、LoRA 和 CLIP,一键全自动连线发布到画布! + +--- + +### 📦 安装指南 + +打开命令行终端,进入 ComfyUI 的 `custom_nodes` 文件夹,执行以下命令: +```bash +cd custom_nodes +git clone https://github.com/DemonGatanjieu/Anomalous_Model_Browser.git +``` +> **注意**:克隆完成后,重启 ComfyUI 即可。您也可以直接在 ComfyUI Manager(管理器)中搜索 `Anomalous Model Browser` 并一键点击安装! diff --git a/ComfyUI/custom_nodes/Anomalous_Model_Browser/__init__.py b/ComfyUI/custom_nodes/Anomalous_Model_Browser/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7c3ee42f2988bf82d99b4ffef6a15b46fe48d44a --- /dev/null +++ b/ComfyUI/custom_nodes/Anomalous_Model_Browser/__init__.py @@ -0,0 +1,12 @@ +import os +from .api import setup_routes +from server import PromptServer + +WEB_DIRECTORY = "./web" + +# 注册 API 路由 +setup_routes(PromptServer.instance.app) + +NODE_CLASS_MAPPINGS = {} +NODE_DISPLAY_NAME_MAPPINGS = {} +__all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS', 'WEB_DIRECTORY'] diff --git a/ComfyUI/custom_nodes/Anomalous_Model_Browser/__pycache__/__init__.cpython-312.pyc b/ComfyUI/custom_nodes/Anomalous_Model_Browser/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f2fa73aac2fd62957fecb039e0b12b3db6db8981 Binary files /dev/null and b/ComfyUI/custom_nodes/Anomalous_Model_Browser/__pycache__/__init__.cpython-312.pyc differ diff --git a/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__init__.py b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b150def270c72ee74445ac2693a052d23cfed011 --- /dev/null +++ b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__init__.py @@ -0,0 +1,43 @@ + +from .models import * +from .scanner import * +from .config import * +from .notebooks import * +from .utils import * + +def setup_routes(app): + app.router.add_get('/anomalous/folders', api_get_folders) + app.router.add_get('/anomalous/all_folder_types', api_get_all_folder_types) + app.router.add_get('/anomalous/models', api_get_models) + app.router.add_get('/anomalous/all_scan_models', api_get_all_scan_models) + app.router.add_get('/anomalous/batch_select', api_batch_select) + app.router.add_get('/anomalous/image', api_serve_image) + app.router.add_post('/anomalous/scan', api_scan_folder) + app.router.add_get('/anomalous/scan_status', api_scan_status) + app.router.add_get('/anomalous/find_model', api_find_model) + app.router.add_get('/anomalous/config', api_get_config) + app.router.add_post('/anomalous/save_config', api_save_config) + app.router.add_post('/anomalous/delete_model', api_delete_model) + app.router.add_post('/anomalous/clean_civitai_info', api_clean_civitai_info) + app.router.add_get('/anomalous/compatible_models', api_compatible_models) + app.router.add_get('/anomalous/notebooks', api_get_notebooks) + app.router.add_post('/anomalous/save_notebook', api_save_notebook) + app.router.add_post('/anomalous/delete_notebook', api_delete_notebook) + app.router.add_post('/anomalous/translate', api_translate) + app.router.add_get('/anomalous/base_models', api_base_models) + app.router.add_get('/anomalous/gallery_images', api_get_gallery_images) + app.router.add_post('/anomalous/delete_gallery_image', api_delete_gallery_image) + app.router.add_get('/anomalous/resolve_hash', api_resolve_hash) + app.router.add_post('/anomalous/resolve_hash_batch', api_resolve_hash_batch) + app.router.add_get('/anomalous/all_hashes', api_get_all_hashes) + app.router.add_post('/anomalous/scan_all', api_scan_all) + app.router.add_get('/anomalous/global_scan_status', api_global_scan_status) + app.router.add_get('/anomalous/scan_missing_models_status', api_scan_missing_models_status) + app.router.add_post('/anomalous/clear_cache', api_clear_cache) + app.router.add_post('/anomalous/update_metadata', api_update_metadata) + app.router.add_post('/anomalous/set_custom_cover', api_set_custom_cover) + app.router.add_post('/anomalous/upload_custom_cover', api_upload_custom_cover) + app.router.add_get('/anomalous/model_images', api_get_model_images) + app.router.add_post('/anomalous/resolve_paths_to_previews', api_resolve_paths_to_previews) + app.router.add_post('/anomalous/scan_missing_models', api_scan_missing_models) + diff --git a/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/__init__.cpython-312.pyc b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..530189980acae6db6c9e79038c771e8b057d4efd Binary files /dev/null and b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/__init__.cpython-312.pyc differ diff --git a/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/config.cpython-312.pyc b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2a1f03129df80b8ddb9b2a85ebdeaa2c8bcae8e4 Binary files /dev/null and b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/config.cpython-312.pyc differ diff --git a/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/metadata.cpython-312.pyc b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/metadata.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b71dd01f0ae33017cb4ce4be5185fbe355d0af43 Binary files /dev/null and b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/metadata.cpython-312.pyc differ diff --git a/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/models.cpython-312.pyc b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/models.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7fb77fdd45b34230c81deb4ac2704e3d9233faf5 Binary files /dev/null and b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/models.cpython-312.pyc differ diff --git a/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/notebooks.cpython-312.pyc b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/notebooks.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6be83dfdbfdea6cab8f6a15d7fb57141de03facf Binary files /dev/null and b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/notebooks.cpython-312.pyc differ diff --git a/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/scanner.cpython-312.pyc b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/scanner.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4da8fb7000ee6f7dc53e1e0120508c9f246e5659 Binary files /dev/null and b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/scanner.cpython-312.pyc differ diff --git a/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/utils.cpython-312.pyc b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..38efe18214654aef4e817c1cda826f1cb03f41db Binary files /dev/null and b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/__pycache__/utils.cpython-312.pyc differ diff --git a/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/config.py b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/config.py new file mode 100644 index 0000000000000000000000000000000000000000..0c5b93e43038eb921039de96cc6a2b1707bec8fc --- /dev/null +++ b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/config.py @@ -0,0 +1,83 @@ +import os +import sys +import json +import urllib.parse +import subprocess +import threading +import asyncio +from aiohttp import web +import folder_paths +import struct + +async def api_get_config(request): + script_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(script_dir, "config.json") + has_key = False + folder_types = [] + if os.path.exists(config_path): + try: + with open(config_path, 'r', encoding='utf-8') as f: + cfg = json.load(f) + has_key = bool(cfg.get("CIVITAI_API_KEY", "").strip()) + folder_types = cfg.get("folder_types_config", []) + except: + pass + if not has_key: + legacy_config_path = os.path.join(os.path.dirname(script_dir), "config.json") + if os.path.exists(legacy_config_path): + try: + with open(legacy_config_path, 'r', encoding='utf-8') as f: + legacy_cfg = json.load(f) + has_key = bool(legacy_cfg.get("CIVITAI_API_KEY", "").strip()) + except Exception: + pass + return web.json_response({"has_api_key": has_key, "folder_types_config": folder_types}) + +async def api_save_config(request): + try: + data = await request.json() + api_key = data.get("api_key") + folder_types_config = data.get("folder_types_config") + + script_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(script_dir, "config.json") + + cfg = {} + if os.path.exists(config_path): + try: + with open(config_path, 'r', encoding='utf-8') as f: + cfg = json.load(f) + except: + pass + + if api_key is not None and not isinstance(api_key, str): + return web.json_response({"status": "error", "message": "Invalid API key"}, status=400) + if folder_types_config is not None and not isinstance(folder_types_config, list): + return web.json_response({"status": "error", "message": "Invalid folder configuration"}, status=400) + if "physical_folders_config" in data and not isinstance(data.get("physical_folders_config"), list): + return web.json_response({"status": "error", "message": "Invalid physical folder configuration"}, status=400) + if "folder_view_mode" in data and data.get("folder_view_mode") not in {"abstract", "physical"}: + return web.json_response({"status": "error", "message": "Invalid folder view mode"}, status=400) + + if api_key is not None: + cfg["CIVITAI_API_KEY"] = api_key.strip() + + if folder_types_config is not None: + cfg["folder_types_config"] = folder_types_config + + if "physical_folders_config" in data: + cfg["physical_folders_config"] = data.get("physical_folders_config") + + if "folder_view_mode" in data: + cfg["folder_view_mode"] = data.get("folder_view_mode") + + temp_path = config_path + ".tmp" + with open(temp_path, 'w', encoding='utf-8') as f: + json.dump(cfg, f, indent=4) + os.replace(temp_path, config_path) + + return web.json_response({"status": "ok"}) + except Exception as e: + return web.json_response({"status": "error", "message": str(e)}) + + diff --git a/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/metadata.py b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/metadata.py new file mode 100644 index 0000000000000000000000000000000000000000..d53857a782ea734728f96edbd6f1d406bcc58eb8 --- /dev/null +++ b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/metadata.py @@ -0,0 +1,227 @@ +import os +import sys +import json +import urllib.parse +import subprocess +import threading +import asyncio +import copy +from functools import lru_cache +from aiohttp import web +import folder_paths +import struct + +def _read_safetensors_hash(file_path): + """Extract hash from safetensors header metadata (O(1) fast read, no full-file SHA256).""" + try: + with open(file_path, "rb") as f: + header_size_bytes = f.read(8) + if len(header_size_bytes) < 8: + return None + header_size = struct.unpack(' 100 * 1024 * 1024: + return None + + header_json_bytes = f.read(header_size) + header_str = header_json_bytes.decode('utf-8') + header_json = json.loads(header_str) + + metadata = header_json.get('__metadata__', {}) + if not metadata: + return None + + if 'modelspec.hash.sha256' in metadata: + return metadata['modelspec.hash.sha256'] + if 'modelspec.hash.blake3' in metadata: + return metadata['modelspec.hash.blake3'] + + except Exception: + pass + return None + + +@lru_cache(maxsize=4096) +def _extract_safetensors_hash_cached(file_path, signature): + return _read_safetensors_hash(file_path) + + +def _extract_safetensors_hash(file_path): + normalized_path = os.path.realpath(file_path) + return _extract_safetensors_hash_cached(normalized_path, _file_signature(normalized_path)) + + +def _select_info_file(data, file_path): + """Select the Civitai file entry that actually describes ``file_path``. + + A model version may contain a checkpoint, text encoder, and VAE in the same + ``files`` array. Taking the first SHA256 silently assigns the dependency's + hash to every local sidecar copied from that version. Physical byte size is + the strongest cheap discriminator available here and remains valid after a + local rename. Filenames are intentionally excluded: local renames are the + reason provenance recovery exists and cannot be identity evidence. + """ + entries = [] + for file_info in data.get("files", []): + if not isinstance(file_info, dict): + continue + hashes = file_info.get("hashes", {}) + sha256 = hashes.get("SHA256") if isinstance(hashes, dict) else None + if not sha256: + continue + + size_bytes = None + try: + if file_info.get("sizeKB") is not None: + size_bytes = int(round(float(file_info["sizeKB"]) * 1024)) + except (TypeError, ValueError, OverflowError): + pass + entries.append((file_info, str(sha256), size_bytes)) + + if not entries: + return None + + try: + physical_size = os.path.getsize(file_path) + except OSError: + physical_size = None + + if physical_size is not None: + size_matches = [entry for entry in entries if entry[2] == physical_size] + if len(size_matches) == 1: + return size_matches[0][0] + if len(size_matches) > 1: + return None + + # Offline-generated metadata commonly contains one injected hash without + # Civitai size/name data. It is safe only when there is a single candidate. + if len(entries) == 1: + return entries[0][0] + return None + + +def _select_info_hash(data, file_path): + selected = _select_info_file(data, file_path) + if not selected: + return "" + hashes = selected.get("hashes", {}) + return str(hashes.get("SHA256", "")) if isinstance(hashes, dict) else "" + +def _read_metadata(file_path): + base_path = os.path.splitext(file_path)[0] + metadata = { + "name": os.path.basename(base_path), + "description": "", + "notes": "", + "trainedWords": [], + "baseModel": "", + "civitai_url": "", + "hash": "", + "custom_name": "", + "custom_notes": "" + } + + info_files = [f"{base_path}.info", f"{base_path}.civitai.info"] + for info_file in info_files: + if os.path.exists(info_file): + try: + with open(info_file, 'r', encoding='utf-8') as f: + data = json.load(f) + + name = data.get("name", "") + if "model" in data and isinstance(data["model"], dict): + model_name = data["model"].get("name", "") + name = f"{model_name} - {name}".strip(' -') + + description = data.get("description", "") or "" + if not description and "model" in data and isinstance(data["model"], dict): + description = data["model"].get("description", "") + notes = data.get("notes", "") + trained_words = data.get("trainedWords", []) + base_model = data.get("baseModel", "") + + model_id = data.get("modelId", "") + version_id = data.get("id", "") + civitai_url = "" + if model_id: + # Handle Civitai's new mature content policy + nsfw_level = data.get("nsfwLevel", 1) + is_nsfw = False + if "model" in data and isinstance(data["model"], dict): + is_nsfw = data["model"].get("nsfw", False) + + domain = "civitai.red" if (nsfw_level > 1 or is_nsfw) else "civitai.com" + civitai_url = f"https://{domain}/models/{model_id}" + if version_id: + civitai_url += f"?modelVersionId={version_id}" + + + selected_file = _select_info_file(data, file_path) + hash_val = "" + if selected_file: + hashes = selected_file.get("hashes", {}) + if isinstance(hashes, dict): + hash_val = str(hashes.get("SHA256", "")) + + if name: metadata["name"] = name + if description: metadata["description"] = description + if notes: metadata["notes"] = notes + if trained_words: metadata["trainedWords"] = trained_words + if base_model: metadata["baseModel"] = base_model + if civitai_url: metadata["civitai_url"] = civitai_url + if hash_val: metadata["hash"] = hash_val + if "anomalous_custom_name" in data and data["anomalous_custom_name"]: metadata["custom_name"] = data["anomalous_custom_name"] + if "anomalous_custom_notes" in data and data["anomalous_custom_notes"]: metadata["custom_notes"] = data["anomalous_custom_notes"] + except Exception: + pass + + # Fallback: if no hash found from .info files, try extracting from safetensors header directly + if not metadata["hash"] and file_path.endswith('.safetensors'): + header_hash = _extract_safetensors_hash(file_path) + if header_hash: + metadata["hash"] = header_hash + + return metadata + + +def _file_signature(file_path): + try: + stat = os.stat(file_path) + return (stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns) + except OSError: + return None + + +def _metadata_signature(file_path): + base_path = os.path.splitext(file_path)[0] + return ( + _file_signature(file_path), + _file_signature(f"{base_path}.info"), + _file_signature(f"{base_path}.civitai.info"), + ) + + +@lru_cache(maxsize=4096) +def _get_metadata_cached(file_path, signature): + # ``signature`` is deliberately unused by the parser itself; it is part of + # the cache key and changes whenever the model or either metadata sidecar + # changes. The cache is bounded so removed/renamed models cannot grow it + # without limit. + return _read_metadata(file_path) + + +def get_metadata(file_path): + normalized_path = os.path.realpath(file_path) + signature = _metadata_signature(normalized_path) + # Metadata contains mutable lists. Never expose the cached object directly. + return copy.deepcopy(_get_metadata_cached(normalized_path, signature)) + + +def clear_metadata_cache(): + _get_metadata_cached.cache_clear() + _extract_safetensors_hash_cached.cache_clear() + + +def get_metadata_cache_info(): + return _get_metadata_cached.cache_info() + + diff --git a/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/models.py b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/models.py new file mode 100644 index 0000000000000000000000000000000000000000..f345b9960337adb70beb4f78f50a8608f1bf937c --- /dev/null +++ b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/models.py @@ -0,0 +1,1295 @@ +from .metadata import get_metadata +import os +import sys +import json +import urllib.parse +import subprocess +import threading +import asyncio +import shutil +from aiohttp import web +import folder_paths +import struct +from .utils import get_active_folder_types, get_folder_view_mode, get_active_physical_basenames, require_filename, resolve_folder_subdir, resolve_within + + +# Cover and sidecar lifecycle is intentionally bounded to exact candidate paths. +# Do not replace these checks with a directory-wide glob/walk: model folders can +# contain thousands of files, while this list has a small, constant upper bound. +MEDIA_EXTENSIONS = ('.png', '.jpg', '.jpeg', '.webp', '.gif', '.avif', '.mp4', '.webm', '.mov', '.avi') +MODEL_EXTENSIONS = ('.safetensors', '.ckpt', '.pt', '.bin') +PREVIEW_SUFFIXES = tuple(f'.preview{ext}' for ext in MEDIA_EXTENSIONS) +CIVITAI_BACKUP_SUFFIXES = tuple(f'.civitai_bak{ext}' for ext in MEDIA_EXTENSIONS) +SIDECAR_SUFFIXES = ( + '.info', '.civitai.info', '.json', '.txt', '.yaml', + *MEDIA_EXTENSIONS, + *PREVIEW_SUFFIXES, + *CIVITAI_BACKUP_SUFFIXES, +) + + +def _first_existing_sidecar(base_path, suffixes): + for suffix in suffixes: + candidate = f"{base_path}{suffix}" + if os.path.isfile(candidate): + return candidate, suffix + return None, None + + +def _reset_model_cover(base_path): + """Restore a recoverable cover without destroying the only existing image.""" + backup_path, backup_suffix = _first_existing_sidecar(base_path, CIVITAI_BACKUP_SUFFIXES) + original_path, _ = _first_existing_sidecar(base_path, MEDIA_EXTENSIONS) + preview_paths = [ + f"{base_path}{suffix}" + for suffix in PREVIEW_SUFFIXES + if os.path.isfile(f"{base_path}{suffix}") + ] + + if backup_path: + media_ext = backup_suffix[len('.civitai_bak'):] + restored_path = f"{base_path}.preview{media_ext}" + temp_path = f"{restored_path}.anomalous_tmp" + try: + # Copy first. If the backup cannot be read, the active custom cover + # remains untouched. os.replace then makes the actual restore atomic. + shutil.copy2(backup_path, temp_path) + os.replace(temp_path, restored_path) + for preview_path in preview_paths: + if preview_path != restored_path and os.path.isfile(preview_path): + os.remove(preview_path) + except Exception as exc: + try: + if os.path.isfile(temp_path): + os.remove(temp_path) + except OSError: + pass + return False, 'restore_failed', str(exc) + return True, 'civitai_backup', None + + if original_path: + try: + for preview_path in preview_paths: + os.remove(preview_path) + except Exception as exc: + return False, 'restore_failed', str(exc) + return True, 'original_cover', None + + if preview_paths: + # There is no recoverable source. Keep the current cover instead of + # turning a harmless Reset click into irreversible image loss. + return False, 'preserved_current', 'No Civitai backup or original cover exists.' + + return True, 'no_cover', None + + +def _cache_token(file_path): + try: + return os.stat(file_path).st_mtime_ns + except OSError: + return 0 + + +def _collect_folder_models(target_dir, folder_type, path_idx, rel_subfolder, page, limit): + """Collect one folder in a worker thread with a single directory listing.""" + try: + with os.scandir(target_dir) as iterator: + file_entries = {} + for entry in iterator: + try: + if entry.is_file(): + file_entries[entry.name] = entry + except OSError: + continue + except OSError: + return {"models": [], "total": 0, "page": page, "limit": limit} + + valid_files = sorted( + (name for name in file_entries if name.endswith(('.safetensors', '.ckpt', '.pt'))), + key=str.lower, + ) + total = len(valid_files) + if limit > 0: + start = max(0, (page - 1) * limit) + sliced = valid_files[start:start + limit] + else: + sliced = valid_files + + q_type = urllib.parse.quote(folder_type) + q_idx = str(path_idx) + q_sub = urllib.parse.quote(rel_subfolder.strip('/')) if rel_subfolder and rel_subfolder != '/' else "" + models = [] + for filename in sliced: + file_path = os.path.join(target_dir, filename) + metadata = get_metadata(file_path) + base_name = os.path.splitext(filename)[0] + preview_file = next( + (base_name + suffix for suffix in PREVIEW_SUFFIXES + MEDIA_EXTENSIONS if base_name + suffix in file_entries), + None, + ) + preview_url = "" + if preview_file: + q_file = urllib.parse.quote(preview_file) + try: + preview_version = file_entries[preview_file].stat().st_mtime_ns + except OSError: + preview_version = 0 + preview_url = ( + f"/anomalous/image?type={q_type}&path_idx={q_idx}&subfolder={q_sub}" + f"&filename={q_file}&t={preview_version}" + ) + try: + model_stat = file_entries[filename].stat() + size_bytes = model_stat.st_size + size_mb = round(size_bytes / (1024 * 1024), 2) + except OSError: + size_mb = 0 + size_bytes = 0 + models.append({ + "filename": filename, + "size_mb": size_mb, + "size_bytes": size_bytes, + "metadata": metadata, + "preview_url": preview_url, + "type": folder_type, + "path_idx": path_idx, + "subfolder": rel_subfolder, + }) + return {"models": models, "total": total, "page": page, "limit": limit} + +def _collect_folders(): + mode = get_folder_view_mode() + result = [] + seen_dirs = set() + + if mode == "physical": + active_bns = get_active_physical_basenames() + all_paths_info = [] + + for t in folder_paths.folder_names_and_paths.keys(): + try: + paths = folder_paths.get_folder_paths(t) + if not paths: continue + for path_idx, base_dir in enumerate(paths): + if not os.path.exists(base_dir): continue + real_path = os.path.realpath(base_dir) + if real_path in seen_dirs: continue + seen_dirs.add(real_path) + + bn = os.path.basename(os.path.normpath(base_dir)) + all_paths_info.append({ + "t": t, + "path_idx": path_idx, + "base_dir": base_dir, + "bn": bn + }) + except: + pass + + for target_bn in active_bns: + matched = [p for p in all_paths_info if p["bn"] == target_bn] + if not matched: continue + + for item in matched: + base_dir = item["base_dir"] + tree = {} + for root, dirs, files in os.walk(base_dir): + has_models = any(f.endswith(('.safetensors', '.ckpt', '.pt')) for f in files) + rel = os.path.relpath(root, base_dir) + if rel == '.': + rel = '/' + else: + rel = '/' + rel.replace('\\', '/') + tree[rel] = { + "path": rel, + "name": os.path.basename(root) if rel != '/' else '[Root]', + "has_models": has_models, + "model_count": sum(1 for f in files if f.endswith(('.safetensors', '.ckpt', '.pt'))) + } + + label = item["bn"] + if len(matched) > 1: + label += f" ({item['path_idx'] + 1})" + + result.append({ + "type": item["t"], + "path_idx": item["path_idx"], + "label": label, + "folders": tree + }) + else: + types = get_active_folder_types() + for t in types: + try: + paths = folder_paths.get_folder_paths(t) + except Exception: + continue + if not paths: + continue + + for path_idx, base_dir in enumerate(paths): + if not os.path.exists(base_dir): + continue + real_path = os.path.realpath(base_dir) + if real_path in seen_dirs: + continue + seen_dirs.add(real_path) + + tree = {} + for root, dirs, files in os.walk(base_dir): + has_models = any(f.endswith(('.safetensors', '.ckpt', '.pt')) for f in files) + rel = os.path.relpath(root, base_dir) + if rel == '.': + rel = '/' + else: + rel = '/' + rel.replace('\\', '/') + tree[rel] = { + "path": rel, + "name": os.path.basename(root) if rel != '/' else '[Root]', + "has_models": has_models, + "model_count": sum(1 for f in files if f.endswith(('.safetensors', '.ckpt', '.pt'))) + } + + try: + folder_basename = os.path.basename(os.path.normpath(base_dir)) + if not folder_basename: + folder_basename = t + except: + folder_basename = t + + label = folder_basename + + basenames = [] + try: + basenames = [os.path.basename(os.path.normpath(p)) for p in paths] + except: + pass + if basenames.count(folder_basename) > 1: + label += f" ({path_idx + 1})" + + result.append({ + "type": t, + "path_idx": path_idx, + "label": label, + "folders": tree + }) + + return {"folders": result} + + +async def api_get_folders(request): + return web.json_response(await asyncio.to_thread(_collect_folders)) + + +async def api_get_models(request): + folder_type = request.query.get('type', 'checkpoints') + subfolder = request.query.get('subfolder', '/') + page = int(request.query.get('page', 1)) + limit = int(request.query.get('limit', 0)) + try: + path_idx = int(request.query.get('path_idx', 0)) + except: + path_idx = 0 + try: + paths = folder_paths.get_folder_paths(folder_type) + except Exception: + return web.json_response({"models": [], "total": 0}) + try: + base_dir, target_dir = resolve_folder_subdir(folder_type, path_idx, subfolder) + except (ValueError, KeyError): + return web.Response(status=400, text='Invalid subfolder') + rel_subfolder = "" if subfolder == '/' else subfolder.strip('/\\') + if not os.path.exists(target_dir): + return web.json_response({"models": [], "total": 0}) + payload = await asyncio.to_thread( + _collect_folder_models, + target_dir, + folder_type, + path_idx, + rel_subfolder, + page, + limit, + ) + return web.json_response(payload) + +async def api_find_model(request): + search = request.query.get('search', '').lower() + if not search: + return web.json_response({"status": "error", "message": "No search query provided"}) + + for folder_type in folder_paths.folder_names_and_paths.keys(): + paths = folder_paths.get_folder_paths(folder_type) + if not paths: continue + for path_idx, base_dir in enumerate(paths): + if not os.path.exists(base_dir): continue + for root, dirs, files in os.walk(base_dir): + for f in files: + if f.endswith('.safetensors') or f.endswith('.ckpt') or f.endswith('.pt') or f.endswith('.bin') or f.endswith('.sft'): + search_norm = search.replace(os.sep, '/') + abs_path_norm = os.path.join(root, f).replace(os.sep, '/').lower() + if search_norm in f.lower() or search_norm in abs_path_norm: + rel_subfolder = os.path.relpath(root, base_dir) + if rel_subfolder == '.': rel_subfolder = '/' + + file_path = os.path.join(root, f) + meta = get_metadata(file_path) + + base_name = os.path.splitext(f)[0] + preview_file = None + for ext in PREVIEW_SUFFIXES + MEDIA_EXTENSIONS: + if os.path.exists(os.path.join(root, base_name + ext)): + preview_file = base_name + ext + break + + preview_url = "" + if preview_file: + q_type = urllib.parse.quote(folder_type) + q_idx = str(path_idx) + q_sub = urllib.parse.quote(rel_subfolder) + q_file = urllib.parse.quote(preview_file) + mtime = _cache_token(os.path.join(root, preview_file)) + preview_url = f"/anomalous/image?type={q_type}&path_idx={q_idx}&subfolder={q_sub}&filename={q_file}&t={mtime}" + + try: size_mb = round(os.path.getsize(file_path) / (1024 * 1024), 1) + except: size_mb = 0 + + modelData = { + "filename": f, + "size_mb": size_mb, + "metadata": meta, + "preview_url": preview_url + } + return web.json_response({ + "status": "success", + "model": modelData, + "type": folder_type, + "path_idx": path_idx, + "subfolder": rel_subfolder + }) + + return web.json_response({"status": "error", "message": "Model not found"}) + +async def api_delete_model(request): + try: + data = await request.json() + folder_type = data.get('type', 'checkpoints') + subfolder = data.get('subfolder', '/') + filename = data.get('filename', '') + try: + path_idx = int(data.get('path_idx', 0)) + except: + path_idx = 0 + + try: + filename = require_filename(filename) + _, target_dir = resolve_folder_subdir(folder_type, path_idx, subfolder) + model_path = resolve_within(target_dir, filename) + except (ValueError, KeyError): + return web.json_response({"status": "error", "message": "Invalid request parameters"}, status=400) + if not os.path.exists(model_path): + return web.json_response({"status": "error", "message": "Model file not found"}) + + # 1. 优先尝试删除你点击的主模型文件 + try: + os.remove(model_path) + except Exception as e: + error_msg = str(e) + if "being used" in error_msg or "WinError 32" in error_msg or "Permission" in error_msg: + error_msg = "文件被占用 (正在被 ComfyUI 使用)。请先重启 ComfyUI 或在工作流中卸载该模型后再删除!" + return web.json_response({"status": "error", "message": f"主模型删除失败: {error_msg}"}) + + base_name = os.path.splitext(filename)[0] + + # 2. 主模型成功删除后,再清理配套的垃圾文件 + # Sidecars are keyed by stem, not by the main model extension. If a + # second real model shares this stem, preserve the shared sidecars for + # the survivor instead of treating that model as cleanup debris. + shared_stem_in_use = any( + os.path.isfile(os.path.join(target_dir, base_name + model_ext)) + for model_ext in MODEL_EXTENSIONS + ) + deleted_files = [filename] + if not shared_stem_in_use: + for suffix in SIDECAR_SUFFIXES: + file_to_del = os.path.join(target_dir, base_name + suffix) + if os.path.isfile(file_to_del): + try: + os.remove(file_to_del) + deleted_files.append(base_name + suffix) + except Exception as e: + print(f"[Anomalous Browser] Warning: Failed to delete {file_to_del}: {e}") + + # 3. 修正:前端期待的成功状态是 "success" 而不是 "ok" + return web.json_response({ + "status": "success", + "deleted": deleted_files, + "sidecars_preserved": shared_stem_in_use, + }) + + except Exception as e: + import traceback + traceback.print_exc() + return web.json_response({"status": "error", "message": str(e)}, status=500) + +async def api_compatible_models(request): + base_model = request.query.get('base_model', '') + target_type = request.query.get('target_type', 'loras') + + if not base_model: + return web.json_response({"models": []}) + + target_types = [t.strip() for t in target_type.split(',')] + compatible_models = [] + seen_files = set() + + for t in target_types: + try: + paths = folder_paths.get_folder_paths(t) + except Exception: + continue + + if not paths: + continue + + for path_idx, base_dir in enumerate(paths): + if not os.path.exists(base_dir): + continue + + for root, _, files in os.walk(base_dir): + for f in files: + if f.endswith('.safetensors') or f.endswith('.ckpt') or f.endswith('.pt'): + file_path = os.path.join(root, f) + real_path = os.path.realpath(file_path) + if real_path in seen_files: + continue + seen_files.add(real_path) + + meta = get_metadata(file_path) + m_bm = str(meta.get("baseModel", "")).strip().lower().replace(" ", "") + req_bm = str(base_model).strip().lower().replace(" ", "") + + if req_bm and m_bm and (req_bm in m_bm or m_bm in req_bm): + rel_subfolder = os.path.relpath(root, base_dir) + if rel_subfolder == '.': + rel_subfolder = '/' + else: + rel_subfolder = '/' + rel_subfolder.replace('\\', '/') + + base_name = os.path.splitext(f)[0] + preview_file = None + for ext in PREVIEW_SUFFIXES + MEDIA_EXTENSIONS: + if os.path.exists(os.path.join(root, base_name + ext)): + preview_file = base_name + ext + break + + preview_url = "" + if preview_file: + q_type = urllib.parse.quote(t) + q_idx = str(path_idx) + q_sub = urllib.parse.quote(rel_subfolder.strip('/')) if rel_subfolder != '/' else "" + q_file = urllib.parse.quote(preview_file) + preview_url = f"/anomalous/image?type={q_type}&path_idx={q_idx}&subfolder={q_sub}&filename={q_file}" + + try: + size_bytes = os.path.getsize(file_path) + size_mb = round(size_bytes / (1024 * 1024), 1) + except Exception: + size_mb = 0 + size_bytes = 0 + + compatible_models.append({ + "type": t, + "path_idx": path_idx, + "subfolder": rel_subfolder, + "filename": f, + "size_mb": size_mb, + "size_bytes": size_bytes, + "preview_url": preview_url, + "metadata": meta + }) + + return web.json_response({"models": compatible_models}) + +async def api_base_models(request): + target_types = get_active_folder_types() + base_models = set() + seen_files = set() + + for t in target_types: + try: + paths = folder_paths.get_folder_paths(t) + except Exception: + continue + if not paths: continue + + for base_dir in paths: + if not os.path.exists(base_dir): continue + for root, _, files in os.walk(base_dir): + for f in files: + if f.endswith('.safetensors') or f.endswith('.ckpt') or f.endswith('.pt'): + file_path = os.path.join(root, f) + real_path = os.path.realpath(file_path) + if real_path in seen_files: continue + seen_files.add(real_path) + + meta = get_metadata(file_path) + m_bm = meta.get("baseModel", "") + if m_bm and str(m_bm).strip(): + # Remove typical generic strings that might pollute + clean_bm = str(m_bm).strip() + base_models.add(clean_bm) + + return web.json_response({"base_models": sorted(list(base_models))}) + +RESOLVABLE_MODEL_TYPES = ('checkpoints', 'loras', 'unet', 'diffusion_models', 'controlnet', 'vae') + + +def _parse_resolution_types(expected_types_raw): + if not expected_types_raw: + return RESOLVABLE_MODEL_TYPES + requested = tuple(dict.fromkeys( + value.strip() for value in str(expected_types_raw).split(',') if value.strip() + )) + if not requested or any(value not in RESOLVABLE_MODEL_TYPES for value in requested): + raise ValueError("Invalid model type") + return requested + + +def _collect_resolution_candidates(types): + candidates = [] + seen_realpaths = set() + for folder_type in types: + try: + paths = folder_paths.get_folder_paths(folder_type) + except Exception: + continue + for base_dir in paths or []: + if not os.path.exists(base_dir): + continue + for root, _, files in os.walk(base_dir): + for filename in files: + if not filename.lower().endswith(('.safetensors', '.ckpt', '.pt')): + continue + file_path = os.path.join(root, filename) + real_path = os.path.realpath(file_path) + if real_path in seen_realpaths: + continue + try: + file_size = os.path.getsize(file_path) + except OSError: + continue + seen_realpaths.add(real_path) + candidates.append({ + "type": folder_type, + "filename": os.path.relpath(file_path, base_dir).replace('\\', '/'), + "path": file_path, + "size": file_size, + }) + return candidates + + +def _candidate_hashes(candidate): + from .metadata import _extract_safetensors_hash + + if "hashes" in candidate: + return candidate["hashes"] + values = set() + metadata = candidate.get("metadata") + if metadata is None: + metadata = get_metadata(candidate["path"]) + candidate["metadata"] = metadata + meta_hash = metadata.get("hash", "") + if meta_hash: + values.add(str(meta_hash).upper()) + if candidate["path"].lower().endswith('.safetensors'): + header_hash = _extract_safetensors_hash(candidate["path"]) + if header_hash: + values.add(str(header_hash).upper()) + candidate["hashes"] = values + return values + + +def _resolved_payload(candidate, **details): + payload = { + "found": True, + "type": candidate["type"], + "filename": candidate["filename"], + } + payload.update(details) + return payload + + +def _resolve_from_candidates(candidates, target_hash="", target_size=None, filename_query=""): + target_hash = str(target_hash or "").strip().upper() + if filename_query: + normalized_query = str(filename_query).replace('\\', '/').lstrip('./') + query_basename = normalized_query.rsplit('/', 1)[-1] + for candidate in candidates: + if candidate["filename"] == normalized_query or ( + '/' not in normalized_query and os.path.basename(candidate["filename"]) == query_basename + ): + return _resolved_payload(candidate, matched_by_filename=True) + + has_target_hash = bool(target_hash and target_hash != "UNKNOWN") + size_matches = [ + candidate for candidate in candidates + if target_size is not None and candidate["size"] == target_size + ] + + if has_target_hash and target_size is not None: + combined_matches = [candidate for candidate in size_matches if target_hash in _candidate_hashes(candidate)] + if len(combined_matches) == 1: + return _resolved_payload(combined_matches[0], matched_by_hash=True, matched_by_size=True) + if len(combined_matches) > 1: + return {"found": False, "ambiguous": True} + + hash_matches = [candidate for candidate in candidates if target_hash in _candidate_hashes(candidate)] + if hash_matches: + return {"found": False, "identity_conflict": True} + if len(size_matches) == 1: + return _resolved_payload(size_matches[0], matched_by_size=True, stale_hash=True) + if len(size_matches) > 1: + return {"found": False, "ambiguous": True} + + if has_target_hash: + hash_matches = [candidate for candidate in candidates if target_hash in _candidate_hashes(candidate)] + if len(hash_matches) == 1: + return _resolved_payload(hash_matches[0], matched_by_hash=True) + if len(hash_matches) > 1: + return {"found": False, "ambiguous": True} + + if target_size is not None: + if len(size_matches) == 1: + return _resolved_payload(size_matches[0], matched_by_size=True) + if len(size_matches) > 1: + return {"found": False, "ambiguous": True} + return {"found": False} + + +async def api_resolve_hash(request): + target_hash = request.query.get("hash", "").strip().upper() + size_str = request.query.get("size", "").strip() + filename_query = request.query.get("filename", "").strip() + target_size = int(size_str) if size_str.isdigit() else None + if not target_hash and target_size is None and not filename_query: + return web.json_response({"found": False}) + try: + types = _parse_resolution_types(request.query.get("type", "").strip()) + except ValueError as exc: + return web.json_response({"found": False, "error": str(exc)}, status=400) + + def resolve_one(): + candidates = _collect_resolution_candidates(types) + return _resolve_from_candidates(candidates, target_hash, target_size, filename_query) + + return web.json_response(await asyncio.to_thread(resolve_one)) + + +async def api_resolve_hash_batch(request): + try: + data = await request.json() + items = data.get("items", []) + except Exception: + return web.json_response({"error": "Invalid JSON body"}, status=400) + if not isinstance(items, list) or len(items) > 256: + return web.json_response({"error": "items must be a list with at most 256 entries"}, status=400) + + parsed_items = [] + try: + for index, item in enumerate(items): + if not isinstance(item, dict): + raise ValueError("Invalid batch item") + size_value = item.get("size") + size_string = str(size_value).strip() if size_value is not None else "" + parsed_items.append({ + "key": str(item.get("key", index)), + "hash": str(item.get("hash", "")).strip().upper(), + "size": int(size_string) if size_string.isdigit() else None, + "types": _parse_resolution_types(str(item.get("type", "")).strip()), + }) + except (TypeError, ValueError) as exc: + return web.json_response({"error": str(exc)}, status=400) + + def resolve_batch(): + candidate_groups = {} + results = [] + for item in parsed_items: + types = item["types"] + if types not in candidate_groups: + candidate_groups[types] = _collect_resolution_candidates(types) + result = _resolve_from_candidates( + candidate_groups[types], + item["hash"], + item["size"], + ) + results.append({"key": item["key"], "result": result}) + return results + + return web.json_response({"results": await asyncio.to_thread(resolve_batch)}) + +async def api_get_all_hashes(request): + """ + Returns a dictionary of all scanned models with their hash and size. + Keyed by both relative path and basename for maximum frontend resilience. + """ + import asyncio + + def fetch_all(): + hashes = {} + ambiguous_keys = set() + + def add_hash(key, value): + if key in ambiguous_keys: + return + existing = hashes.get(key) + if existing is None or existing == value: + hashes[key] = value + else: + hashes.pop(key, None) + ambiguous_keys.add(key) + + types = ['checkpoints', 'loras', 'unet', 'diffusion_models', 'controlnet', 'vae'] + seen_dirs = set() + for t in types: + try: + paths = folder_paths.get_folder_paths(t) + if not paths: continue + for base_dir in paths: + if base_dir in seen_dirs: continue + seen_dirs.add(base_dir) + if not os.path.exists(base_dir): continue + for root, dirs, files in os.walk(base_dir): + for file in files: + if file.endswith('.safetensors') or file.endswith('.ckpt') or file.endswith('.pt'): + file_path = os.path.join(root, file) + try: + size_bytes = os.path.getsize(file_path) + except Exception: + size_bytes = 0 + + meta = get_metadata(file_path) + hash_val = "" + if meta and meta.get("hash"): + hash_val = meta["hash"] + + rel_path = os.path.relpath(file_path, base_dir) + if rel_path.startswith('.\\') or rel_path.startswith('./'): + rel_path = rel_path[2:] + rel_path = rel_path.replace('\\', '/') + basename = os.path.basename(file_path) + + val = {"hash": hash_val, "size": size_bytes} + add_hash(rel_path, val) + add_hash(basename, val) + except Exception: + pass + return hashes + + hashes = await asyncio.to_thread(fetch_all) + return web.json_response(hashes) + +async def api_update_metadata(request): + try: + data = await request.json() + folder_type = data.get('type', 'checkpoints') + subfolder = data.get('subfolder', '/') + filename = data.get('filename', '') + custom_name = data.get('custom_name', '') + custom_notes = data.get('custom_notes', '') + physical_rename = data.get('physical_rename', False) + try: path_idx = int(data.get('path_idx', 0)) + except: path_idx = 0 + + try: + filename = require_filename(filename) + _, target_dir = resolve_folder_subdir(folder_type, path_idx, subfolder) + file_path = resolve_within(target_dir, filename) + except (ValueError, KeyError): + return web.json_response({"status": "error", "message": "Invalid request parameters"}, status=400) + + if not os.path.exists(file_path): + return web.json_response({"status": "error", "message": "Model not found"}) + + base_name = os.path.splitext(file_path)[0] + model_ext = os.path.splitext(file_path)[1] + info_file = f"{base_name}.civitai.info" + + info_data = {} + if os.path.exists(info_file): + parsed = False + for enc in ['utf-8', 'utf-8-sig', 'mbcs', 'latin-1']: + try: + with open(info_file, 'r', encoding=enc) as f: + info_data = json.load(f) + parsed = True + break + except Exception: + pass + if not parsed: + return web.json_response({"status": "error", "message": "Failed to parse existing .civitai.info file due to encoding or corruption. Rename aborted to prevent data loss."}) + + info_data["anomalous_custom_name"] = custom_name + info_data["anomalous_custom_notes"] = custom_notes + + reset_cover = data.get('reset_cover', False) + cover_reset = None + cover_reset_source = None + cover_reset_warning = None + if reset_cover: + cover_reset, cover_reset_source, cover_reset_warning = _reset_model_cover(base_name) + + with open(info_file, 'w', encoding='utf-8') as f: + json.dump(info_data, f, indent=4, ensure_ascii=False) + + new_filename = filename + + if physical_rename and custom_name: + import re + safe_name = re.sub(r'[<>:"/\\|?*]', '_', custom_name).strip(' .') + if not safe_name: + return web.json_response({"status": "error", "message": "The physical filename cannot be empty."}, status=400) + new_file_path = os.path.join(target_dir, f"{safe_name}{model_ext}") + + if new_file_path != file_path and not os.path.exists(new_file_path): + os.rename(file_path, new_file_path) + + for suffix in SIDECAR_SUFFIXES: + old_sidecar = f"{base_name}{suffix}" + if os.path.isfile(old_sidecar): + os.rename(old_sidecar, os.path.join(target_dir, f"{safe_name}{suffix}")) + + new_filename = f"{safe_name}{model_ext}" + elif os.path.exists(new_file_path) and new_file_path != file_path: + return web.json_response({"status": "error", "message": "A file with the target physical name already exists."}) + + response_data = {"status": "success", "new_filename": new_filename} + if reset_cover: + response_data.update({ + "cover_reset": cover_reset, + "cover_reset_source": cover_reset_source, + "cover_reset_warning": cover_reset_warning, + }) + return web.json_response(response_data) + except Exception as e: + return web.json_response({"status": "error", "message": str(e)}) + +async def _handle_custom_cover(target_dir, filename, save_func, source_ext='.png'): + base_name = os.path.splitext(filename)[0] + + # Always save custom covers with a .preview.[ext] suffix so standard nodes recognize them as covers. + if source_ext.startswith('.preview.'): + preview_ext = source_ext + else: + preview_ext = f".preview{source_ext}" + + dest_path = os.path.join(target_dir, f"{base_name}{preview_ext}") + + # Delete any existing .preview.* files to ensure only one custom cover is active + for ext in PREVIEW_SUFFIXES: + p = os.path.join(target_dir, f"{base_name}{ext}") + if os.path.exists(p) and p != dest_path: + try: os.remove(p) + except: pass + + await save_func(dest_path) + +async def api_set_custom_cover(request): + try: + data = await request.json() + folder_type = data.get('type', 'checkpoints') + subfolder = data.get('subfolder', '/') + filename = data.get('filename', '') + source_image = data.get('source_image', '') + try: path_idx = int(data.get('path_idx', 0)) + except: path_idx = 0 + + try: + filename = require_filename(filename) + _, target_dir = resolve_folder_subdir(folder_type, path_idx, subfolder) + output_dir = folder_paths.get_output_directory() + src_path = resolve_within(output_dir, source_image) + except (ValueError, KeyError): + return web.json_response({"status": "error", "message": "Invalid request parameters"}, status=400) + if not os.path.exists(src_path): + return web.json_response({"status": "error", "message": "Source image not found in output directory"}) + + source_ext = os.path.splitext(src_path)[1].lower() + if source_ext not in {'.png', '.jpg', '.jpeg', '.webp', '.gif', '.avif', '.mp4', '.webm', '.mov', '.avi'}: + return web.json_response({"status": "error", "message": "Unsupported cover format"}, status=415) + + async def save_copy(dest_path): + import shutil + import asyncio + await asyncio.to_thread(shutil.copy2, src_path, dest_path) + + await _handle_custom_cover(target_dir, filename, save_copy, source_ext) + + return web.json_response({"status": "success"}) + except Exception as e: + return web.json_response({"status": "error", "message": str(e)}) + +async def api_upload_custom_cover(request): + try: + data = await request.post() + folder_type = data.get('type', 'checkpoints') + subfolder = data.get('subfolder', '/') + filename = data.get('filename', '') + try: path_idx = int(data.get('path_idx', 0)) + except: path_idx = 0 + + image_field = data.get('image') + + try: + filename = require_filename(filename) + _, target_dir = resolve_folder_subdir(folder_type, path_idx, subfolder) + except (ValueError, KeyError): + return web.json_response({"status": "error", "message": "Invalid request parameters"}, status=400) + if image_field is None: + return web.json_response({"status": "error", "message": "Image is required"}, status=400) + + image_data = image_field.file.read() + if len(image_data) > 100 * 1024 * 1024: + return web.json_response({"status": "error", "message": "Cover file is too large"}, status=413) + + upload_filename = image_field.filename + source_ext = os.path.splitext(upload_filename)[1].lower() + if source_ext not in {'.png', '.jpg', '.jpeg', '.webp', '.gif', '.avif', '.mp4', '.webm', '.mov', '.avi'}: + return web.json_response({"status": "error", "message": "Unsupported cover format"}, status=415) + + async def save_upload(dest_path): + def write_file(): + with open(dest_path, 'wb') as f: + f.write(image_data) + import asyncio + await asyncio.to_thread(write_file) + + await _handle_custom_cover(target_dir, filename, save_upload, source_ext) + + return web.json_response({"status": "success"}) + except Exception as e: + return web.json_response({"status": "error", "message": str(e)}) + + +import struct + + +def _preview_url_for_model(folder_type, path_idx, base_dir, file_path): + root = os.path.dirname(file_path) + filename = os.path.basename(file_path) + base_name = os.path.splitext(filename)[0] + preview_file = next( + ( + base_name + suffix + for suffix in PREVIEW_SUFFIXES + MEDIA_EXTENSIONS + if os.path.isfile(os.path.join(root, base_name + suffix)) + ), + None, + ) + if not preview_file: + return "" + rel_subfolder = os.path.relpath(root, base_dir) + if rel_subfolder == '.': + rel_subfolder = '/' + q_type = urllib.parse.quote(folder_type) + q_idx = str(path_idx) + q_sub = urllib.parse.quote(rel_subfolder) + q_file = urllib.parse.quote(preview_file) + version = _cache_token(os.path.join(root, preview_file)) + return f"/anomalous/image?type={q_type}&path_idx={q_idx}&subfolder={q_sub}&filename={q_file}&t={version}" + + +def _model_info_for_path(folder_type, path_idx, base_dir, file_path): + root = os.path.dirname(file_path) + rel_subfolder = os.path.relpath(root, base_dir) + if rel_subfolder == '.': + rel_subfolder = '/' + else: + rel_subfolder = '/' + rel_subfolder.replace(os.sep, '/') + return { + "type": folder_type, + "path_idx": path_idx, + "subfolder": rel_subfolder, + "filename": os.path.basename(file_path), + "preview_url": _preview_url_for_model(folder_type, path_idx, base_dir, file_path), + "metadata": get_metadata(file_path), + } + + +def _allowed_folder_types(requested_types=None): + available = list(folder_paths.folder_names_and_paths.keys()) + if requested_types is None: + return available + if not isinstance(requested_types, list): + return [] + allowed = set(available) + return list(dict.fromkeys( + folder_type + for folder_type in requested_types + if isinstance(folder_type, str) and folder_type in allowed + )) + + +def _resolve_paths_to_model_info_sync(paths, folder_types=None): + requested = [ + (path, path.replace('\\', '/').lower(), path.replace('\\', '/')) + for path in paths + if isinstance(path, str) + ] + exact_results = {} + roots = [] + for folder_type in _allowed_folder_types(folder_types): + try: + folder_dirs = folder_paths.get_folder_paths(folder_type) + except Exception: + continue + for path_idx, base_dir in enumerate(folder_dirs or []): + if not os.path.isdir(base_dir): + continue + real_base_dir = os.path.realpath(base_dir) + roots.append((folder_type, path_idx, real_base_dir)) + for original, _, relative_path in requested: + relative = relative_path.replace('/', os.sep) + candidate = os.path.realpath(os.path.join(real_base_dir, relative)) + try: + if os.path.commonpath((real_base_dir, candidate)) != real_base_dir: + continue + except ValueError: + continue + if os.path.isfile(candidate) and candidate.lower().endswith(('.safetensors', '.ckpt', '.pt', '.bin', '.sft')): + exact_results[original] = _model_info_for_path(folder_type, path_idx, base_dir, candidate) + + unresolved = [(original, normalized) for original, normalized, _ in requested if original not in exact_results] + if not unresolved: + return exact_results + + wanted_relpaths = {normalized for _, normalized in unresolved} + wanted_basenames = {normalized.rsplit('/', 1)[-1] for _, normalized in unresolved} + rel_matches = {} + basename_matches = {} + for folder_type, path_idx, base_dir in roots: + for root, _, files in os.walk(base_dir): + for filename in files: + if not filename.lower().endswith(('.safetensors', '.ckpt', '.pt', '.bin', '.sft')): + continue + rel_path = os.path.relpath(os.path.join(root, filename), base_dir).replace(os.sep, '/').lower() + basename = filename.lower() + if rel_path not in wanted_relpaths and basename not in wanted_basenames: + continue + candidate_info = _model_info_for_path( + folder_type, + path_idx, + base_dir, + os.path.join(root, filename), + ) + if rel_path in wanted_relpaths: + rel_matches[rel_path] = candidate_info + if basename in wanted_basenames: + basename_matches[basename] = candidate_info + + model_info = dict(exact_results) + for original, normalized in unresolved: + if normalized in rel_matches: + model_info[original] = rel_matches[normalized] + else: + basename = normalized.rsplit('/', 1)[-1] + if basename in basename_matches: + model_info[original] = basename_matches[basename] + return model_info + + +def _resolve_paths_to_previews_sync(paths, folder_types=None): + model_info = _resolve_paths_to_model_info_sync(paths, folder_types) + return {path: item.get("preview_url", "") for path, item in model_info.items()} + + +async def api_resolve_paths_to_previews(request): + try: + data = await request.json() + paths = data.get('paths', []) + folder_types = data.get('folder_types') + context_requests = data.get('context_requests', []) + except: + return web.json_response({"previews": {}, "models": {}, "context_models": {}}) + + model_info = await asyncio.to_thread(_resolve_paths_to_model_info_sync, paths, folder_types) + context_models = {} + if isinstance(context_requests, list): + for item in context_requests[:16]: + if not isinstance(item, dict): + continue + context_path = item.get('path') + if not isinstance(context_path, str) or not context_path: + continue + resolved = await asyncio.to_thread( + _resolve_paths_to_model_info_sync, + [context_path], + item.get('folder_types'), + ) + if context_path in resolved: + context_models[context_path] = resolved[context_path] + + previews = {path: item.get("preview_url", "") for path, item in model_info.items()} + return web.json_response({ + "previews": previews, + "models": model_info, + "context_models": context_models, + }) + + + + +def _collect_all_scan_models(page, limit): + target_types = get_active_folder_types() + all_tuples = [] + seen_dirs = set() + for t in target_types: + try: + paths = folder_paths.get_folder_paths(t) + except Exception: + continue + if not paths: continue + for path_idx, base_dir in enumerate(paths): + if base_dir in seen_dirs: continue + seen_dirs.add(base_dir) + if not os.path.exists(base_dir): continue + for root, dirs, files in os.walk(base_dir): + for f in files: + if f.endswith(('.safetensors', '.ckpt', '.pt', '.bin', '.sft')): + all_tuples.append((t, path_idx, root, base_dir, f)) + + all_tuples.sort(key=lambda x: (x[0], x[4].lower())) + total = len(all_tuples) + + if limit > 0: + start = (page - 1) * limit + end = start + limit + sliced = all_tuples[start:end] + else: + sliced = all_tuples + + all_models = [] + for t, path_idx, root, base_dir, f in sliced: + file_path = os.path.join(root, f) + rel_subfolder = os.path.relpath(root, base_dir) + if rel_subfolder == '.': rel_subfolder = '' + else: rel_subfolder = rel_subfolder.replace('\\', '/') + try: + size_bytes = os.path.getsize(file_path) + size_mb = round(size_bytes / (1024 * 1024), 2) + except: + size_bytes = 0; size_mb = 0 + meta = get_metadata(file_path) + base_name = os.path.splitext(f)[0] + preview_file = None + for ext in PREVIEW_SUFFIXES + MEDIA_EXTENSIONS: + if os.path.exists(os.path.join(root, base_name + ext)): + preview_file = base_name + ext + break + preview_url = "" + if preview_file: + q_type = urllib.parse.quote(t) + q_idx = str(path_idx) + q_sub = urllib.parse.quote(rel_subfolder.strip('/')) if rel_subfolder and rel_subfolder != '/' else "" + q_file = urllib.parse.quote(preview_file) + mtime = _cache_token(os.path.join(root, preview_file)) + preview_url = f"/anomalous/image?type={q_type}&path_idx={q_idx}&subfolder={q_sub}&filename={q_file}&t={mtime}" + all_models.append({ + "type": t, "path_idx": path_idx, "subfolder": rel_subfolder, + "filename": f, "size_mb": size_mb, "size_bytes": size_bytes, + "preview_url": preview_url, "metadata": meta + }) + return {"models": all_models, "total": total, "page": page, "limit": limit} + + +async def api_get_all_scan_models(request): + page = int(request.query.get('page', 1)) + limit = int(request.query.get('limit', 0)) + payload = await asyncio.to_thread(_collect_all_scan_models, page, limit) + return web.json_response(payload) + +async def api_batch_select(request): + folder_key = request.query.get('folderKey', 'ALL') + action = request.query.get('action', 'all') + + def matches_condition(file_path, root, base_name): + if action == 'all': + return True + elif action == 'no_preview': + for ext in PREVIEW_SUFFIXES + MEDIA_EXTENSIONS: + if os.path.exists(os.path.join(root, base_name + ext)): + return False + return True + elif action == 'no_desc': + info_file = file_path + '.info' + civitai_info = os.path.join(root, base_name + '.civitai.info') + if os.path.exists(civitai_info): + try: + with open(civitai_info, 'r', encoding='utf-8') as f: + data = json.load(f) + if data.get('description', '').strip(): + return False + except: pass + if os.path.exists(info_file): + try: + with open(info_file, 'r', encoding='utf-8') as f: + data = json.load(f) + if data.get('description', '').strip(): + return False + except: pass + return True + return False + + results = {} + + if folder_key == 'ALL': + target_types = get_active_folder_types() + seen_dirs = set() + for t in target_types: + try: paths = folder_paths.get_folder_paths(t) + except Exception: continue + if not paths: continue + for path_idx, base_dir in enumerate(paths): + if base_dir in seen_dirs: continue + seen_dirs.add(base_dir) + if not os.path.exists(base_dir): continue + for root, dirs, files in os.walk(base_dir): + for f in files: + if f.endswith(('.safetensors', '.ckpt', '.pt', '.bin', '.sft')): + file_path = os.path.join(root, f) + base_name = os.path.splitext(f)[0] + if matches_condition(file_path, root, base_name): + rel_subfolder = os.path.relpath(root, base_dir) + if rel_subfolder == '.': rel_subfolder = '' + else: rel_subfolder = rel_subfolder.replace('\\', '/') + fkey = f"{t}|{path_idx}|{rel_subfolder}" + if fkey not in results: results[fkey] = [] + results[fkey].append(f) + else: + parts = folder_key.split('|') + if len(parts) >= 3: + t = parts[0] + path_idx = int(parts[1]) + subfolder = parts[2] + + try: paths = folder_paths.get_folder_paths(t) + except Exception: paths = [] + + if paths and path_idx < len(paths): + try: + base_dir, target_dir = resolve_folder_subdir(t, path_idx, subfolder) + except ValueError: + return web.json_response({"selected": {}}) + + if os.path.exists(target_dir): + try: entries = os.listdir(target_dir) + except: entries = [] + for f in entries: + if f.endswith(('.safetensors', '.ckpt', '.pt', '.bin', '.sft')): + file_path = os.path.join(target_dir, f) + if os.path.isfile(file_path): + base_name = os.path.splitext(f)[0] + if matches_condition(file_path, target_dir, base_name): + if folder_key not in results: results[folder_key] = [] + results[folder_key].append(f) + + return web.json_response({"selected": results}) diff --git a/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/notebooks.py b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/notebooks.py new file mode 100644 index 0000000000000000000000000000000000000000..6b663ecd5b5178be2da161923eecdb19065a33f4 --- /dev/null +++ b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/notebooks.py @@ -0,0 +1,77 @@ +import os +import sys +import json +import urllib.parse +import subprocess +import threading +import asyncio +from aiohttp import web +import folder_paths +import struct +from .utils import require_filename, resolve_within + +async def api_get_notebooks(request): + script_dir = os.path.dirname(os.path.abspath(__file__)) + nb_dir = os.path.join(script_dir, "notebooks") + if not os.path.exists(nb_dir): + os.makedirs(nb_dir) + + notebooks = [] + for f in os.listdir(nb_dir): + if f.endswith('.json'): + try: + with open(os.path.join(nb_dir, f), 'r', encoding='utf-8') as file: + data = json.load(file) + notebooks.append({ + "filename": f, + "name": data.get("name", f.replace('.json', '')), + "data": data + }) + except Exception: + pass + return web.json_response({"notebooks": notebooks}) + +async def api_save_notebook(request): + try: + data = await request.json() + filename = data.get("filename", "") + if not filename.endswith('.json'): + filename += '.json' + try: + filename = require_filename(filename) + except ValueError: + return web.json_response({"status": "error", "message": "Invalid filename"}, status=400) + + script_dir = os.path.dirname(os.path.abspath(__file__)) + nb_dir = os.path.join(script_dir, "notebooks") + if not os.path.exists(nb_dir): + os.makedirs(nb_dir) + + file_path = resolve_within(nb_dir, filename) + with open(file_path, 'w', encoding='utf-8') as f: + json.dump(data.get("data", {}), f, indent=4, ensure_ascii=False) + + return web.json_response({"status": "success"}) + except Exception as e: + return web.json_response({"status": "error", "message": str(e)}) + +async def api_delete_notebook(request): + try: + data = await request.json() + filename = data.get("filename", "") + try: + filename = require_filename(filename) + except ValueError: + return web.json_response({"status": "error", "message": "Invalid filename"}, status=400) + + script_dir = os.path.dirname(os.path.abspath(__file__)) + nb_dir = os.path.join(script_dir, "notebooks") + file_path = resolve_within(nb_dir, filename) + + if os.path.exists(file_path): + os.remove(file_path) + return web.json_response({"status": "success"}) + return web.json_response({"status": "error", "message": "File not found"}) + except Exception as e: + return web.json_response({"status": "error", "message": str(e)}) + diff --git a/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/scanner.py b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/scanner.py new file mode 100644 index 0000000000000000000000000000000000000000..c42a9fc45d7da8b2ef57347f1ad5bf2ef26848dd --- /dev/null +++ b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/scanner.py @@ -0,0 +1,404 @@ +import os +import sys +import json +import urllib.parse +import subprocess +import threading +import asyncio +from aiohttp import web +import folder_paths +import struct +from .utils import get_active_folder_types, get_active_scan_paths, resolve_folder_subdir + +async def api_scan_status(request): + folder_type = request.query.get('type', 'checkpoints') + subfolder = request.query.get('subfolder', '/') + try: + path_idx = int(request.query.get('path_idx', 0)) + except: + path_idx = 0 + try: + paths = folder_paths.get_folder_paths(folder_type) + except Exception: + return web.json_response({"scanning": False}) + if not paths or path_idx < 0 or path_idx >= len(paths): + return web.json_response({"scanning": False}) + try: + base_dir, target_dir = resolve_folder_subdir(folder_type, path_idx, subfolder) + except ValueError: + return web.json_response({"scanning": False}) + + marker_file = os.path.join(target_dir, '.scan_in_progress') + result_file = os.path.join(target_dir, '.scan_result.json') + scanning = os.path.exists(marker_file) + + data = {"scanning": scanning} + if not scanning and os.path.exists(result_file): + try: + with open(result_file, 'r', encoding='utf-8') as f: + data["result"] = __import__('json').load(f) + os.remove(result_file) + except: + pass + + return web.json_response(data) + +async def api_scan_folder(request): + """Launches the scraper in the background for a specific directory.""" + folder_type = request.query.get('type', 'checkpoints') + subfolder = request.query.get('subfolder', '/') + try: + path_idx = int(request.query.get('path_idx', 0)) + except: + path_idx = 0 + + try: + base_dir, target_dir = resolve_folder_subdir(folder_type, path_idx, subfolder) + except (ValueError, KeyError): + return web.json_response({"status": "error", "message": "Invalid folder type"}) + + if not os.path.exists(target_dir): + return web.json_response({"status": "error", "message": "Directory does not exist"}) + + # Get the scraper path relative to this script + plugin_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + scraper_path = os.path.join(plugin_dir, "scraper.py") + + if not os.path.exists(scraper_path): + return web.json_response({"status": "error", "message": "scraper.py not found in extension directory"}) + + print(f"[Anomalous Browser] Starting background scan for: {target_dir}") + + try: + data = await request.json() + except Exception: + data = {} + + offline_only = data.get("offline_only", False) + skip_rename = data.get("skip_rename", False) + virtual_rename = data.get("virtual_rename", False) + physical_rename = data.get("physical_rename", False) + force_overwrite = data.get("force_overwrite", False) + + target_files_list = data.get("target_files", []) + if not target_files_list: + target_files_str = request.query.get('target_files', '') + target_files_list = [f.strip() for f in target_files_str.split(',')] if target_files_str else [] + + try: + marker_file = os.path.join(target_dir, '.scan_in_progress') + try: + with open(marker_file, 'x') as f: + f.write('1') + except FileExistsError: + return web.json_response({"status": "error", "message": "Scan already in progress"}, status=409) + + if target_files_list: + targets_file = os.path.join(target_dir, '.scan_targets.json') + with open(targets_file, 'w', encoding='utf-8') as f: + __import__('json').dump(target_files_list, f) + + def run_bg(): + try: + cmd = [sys.executable, scraper_path, target_dir] + if offline_only: + cmd.append("--offline-only") + if skip_rename: + cmd.append("--skip-rename") + if virtual_rename: + cmd.append("--virtual-rename") + if physical_rename: + cmd.append("--physical-rename") + if force_overwrite: + cmd.append("--force-overwrite") + + result = subprocess.run( + cmd, + cwd=plugin_dir + ) + if result.returncode != 0: + result_file = os.path.join(target_dir, '.scan_result.json') + with open(result_file, 'w', encoding='utf-8') as f: + json.dump({"success": 0, "fail": 1, "error": f"Scanner exited with code {result.returncode}"}, f) + finally: + if hasattr(folder_paths, "filename_list_cache"): + try: folder_paths.filename_list_cache.clear() + except: pass + if hasattr(folder_paths, "cache_helper") and hasattr(folder_paths.cache_helper, "clear"): + try: folder_paths.cache_helper.clear() + except: pass + + if os.path.exists(marker_file): + try: os.remove(marker_file) + except: pass + + threading.Thread(target=run_bg, daemon=True).start() + + return web.json_response({"status": "ok", "message": "Scan started in background. Check console for details."}) + except Exception as e: + if 'marker_file' in locals() and os.path.exists(marker_file): + try: os.remove(marker_file) + except OSError: pass + return web.json_response({"status": "error", "message": str(e)}) + +async def api_scan_all(request): + plugin_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + scraper_path = os.path.join(plugin_dir, "scraper.py") + marker_file = os.path.join(plugin_dir, '.global_scan_in_progress') + + try: + data = await request.json() + except Exception: + data = {} + + offline_only = data.get("offline_only", False) + use_local_metadata = data.get("use_local_metadata", True) + skip_rename = data.get("skip_rename", True) + virtual_rename = data.get("virtual_rename", False) + physical_rename = data.get("physical_rename", False) + force_overwrite = data.get("force_overwrite", False) + skip_media = data.get("skip_media", False) + + try: + try: + with open(marker_file, 'x') as f: + f.write('1') + except FileExistsError: + return web.json_response({"status": "error", "message": "Global scan already in progress"}, status=409) + + def run_global_bg(): + try: + paths_to_scan = get_active_scan_paths() + for base_dir in paths_to_scan: + if not os.path.exists(base_dir): continue + try: + print(f"[Anomalous Browser] Global scan processing: {base_dir}") + cmd = [sys.executable, scraper_path, base_dir] + if offline_only: + cmd.append("--offline-only") + if skip_rename: + cmd.append("--skip-rename") + if virtual_rename: + cmd.append("--virtual-rename") + if physical_rename: + cmd.append("--physical-rename") + if force_overwrite: + cmd.append("--force-overwrite") + if skip_media: + cmd.append("--skip-media") + if not use_local_metadata: + cmd.append("--skip-local-metadata") + result = subprocess.run( + cmd, + cwd=plugin_dir + ) + if result.returncode != 0: + print(f"[Anomalous Browser] Scanner exited with code {result.returncode}: {base_dir}") + except Exception as e: + print(f"[Anomalous Browser] Global scan error on {base_dir}: {e}") + finally: + if hasattr(folder_paths, "filename_list_cache"): + try: folder_paths.filename_list_cache.clear() + except: pass + if hasattr(folder_paths, "cache_helper") and hasattr(folder_paths.cache_helper, "clear"): + try: folder_paths.cache_helper.clear() + except: pass + + if os.path.exists(marker_file): + try: os.remove(marker_file) + except: pass + + threading.Thread(target=run_global_bg, daemon=True).start() + return web.json_response({"status": "ok", "message": "Global scan started"}) + except Exception as e: + if os.path.exists(marker_file): + try: os.remove(marker_file) + except OSError: pass + return web.json_response({"status": "error", "message": str(e)}) + +async def api_global_scan_status(request): + plugin_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + marker_file = os.path.join(plugin_dir, '.global_scan_in_progress') + return web.json_response({"scanning": os.path.exists(marker_file)}) +async def api_clean_civitai_info(request): + try: + paths_to_clean = get_active_scan_paths() + deleted_count = 0 + for base_dir in paths_to_clean: + if not os.path.exists(base_dir): continue + for root, dirs, files in os.walk(base_dir): + for file in files: + if file.endswith('.civitai.info'): + file_path = os.path.join(root, file) + try: + os.remove(file_path) + deleted_count += 1 + except Exception as e: + pass + + return web.json_response({"status": "success", "count": deleted_count}) + except Exception as e: + return web.json_response({"status": "error", "message": str(e)}, status=500) + +GLOBAL_SCAN_STATE = { + "scanning": False, + "total": 0, + "current": 0, + "filename": "", + "error": "" +} +GLOBAL_SCAN_LOCK = threading.Lock() + +async def api_scan_missing_models_status(request): + return web.json_response(GLOBAL_SCAN_STATE) + +async def api_scan_missing_models(request): + import sys + import threading + import traceback + + plugin_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + if plugin_dir not in sys.path: + sys.path.insert(0, plugin_dir) + + try: + from scraper import extract_safetensors_hash, calculate_sha256, fetch_civitai_info, infer_base_model_from_header + except ImportError: + return web.json_response({"status": "error", "message": "Failed to load scraper module"}) + + with GLOBAL_SCAN_LOCK: + if GLOBAL_SCAN_STATE["scanning"]: + return web.json_response({"status": "error", "message": "Scan already in progress"}, status=409) + GLOBAL_SCAN_STATE["scanning"] = True + GLOBAL_SCAN_STATE["total"] = 0 + GLOBAL_SCAN_STATE["current"] = 0 + GLOBAL_SCAN_STATE["filename"] = "Initializing..." + GLOBAL_SCAN_STATE["error"] = "" + + try: + data = await request.json() + except Exception: + data = {} + + force_overwrite = data.get("force_overwrite", False) + + def run_deep_scan(): + try: + paths_to_scan = get_active_scan_paths() + files_to_scan = [] + for base_dir in paths_to_scan: + if not os.path.exists(base_dir): continue + for root, dirs, files in os.walk(base_dir): + for file in files: + if not file.endswith('.safetensors'): + continue + file_path = os.path.join(root, file) + base_path = os.path.splitext(file_path)[0] + info_path = base_path + ".info" + civitai_info_path = base_path + ".civitai.info" + + # Check if a valid info file exists + has_valid_info = False + actual_info = info_path if os.path.exists(info_path) else (civitai_info_path if os.path.exists(civitai_info_path) else None) + + if force_overwrite: + actual_info = None + + if actual_info: + try: + with open(actual_info, 'r', encoding='utf-8') as f: + data = json.load(f) + for fi in data.get("files", []): + if isinstance(fi, dict) and "hashes" in fi and "SHA256" in fi["hashes"]: + if fi["hashes"]["SHA256"]: + has_valid_info = True + break + except Exception: + pass + + if not has_valid_info: + files_to_scan.append(file_path) + + GLOBAL_SCAN_STATE["total"] = len(files_to_scan) + + for idx, file_path in enumerate(files_to_scan): + filename = os.path.basename(file_path) + GLOBAL_SCAN_STATE["current"] = idx + 1 + GLOBAL_SCAN_STATE["filename"] = filename + + try: + civitai_data = None + file_hash = None + + # Fallback 1: Try header hash on Civitai + header_hash = extract_safetensors_hash(file_path) + if header_hash: + civitai_data = fetch_civitai_info(header_hash) + if civitai_data: + file_hash = header_hash + + # Fallback 2: If header hash fails, compute full SHA256 + if not civitai_data: + full_hash = calculate_sha256(file_path) + civitai_data = fetch_civitai_info(full_hash) + file_hash = full_hash + + # Fallback 3: Local Offline Inference + if not civitai_data: + inferred_base = infer_base_model_from_header(file_path) + if inferred_base == 'Unknown': + inferred_base = "" + + if not file_hash: + file_hash = header_hash or calculate_sha256(file_path) + + civitai_data = { + "id": -1, + "modelId": -1, + "name": os.path.splitext(filename)[0], + "baseModel": inferred_base, + "description": "

Automatically inferred by Anomalous Local Engine.

", + "model": { + "name": os.path.splitext(filename)[0], + "type": "LORA" if "lora" in file_path.lower() else "Checkpoint" + }, + "files": [{"hashes": {"SHA256": file_hash}}] + } + + # Ensure SHA256 is explicitly injected into the info data + if civitai_data: + if not civitai_data.get("files"): + civitai_data["files"] = [{"hashes": {"SHA256": file_hash}}] + elif isinstance(civitai_data["files"][0], dict): + if "hashes" not in civitai_data["files"][0]: + civitai_data["files"][0]["hashes"] = {} + civitai_data["files"][0]["hashes"]["SHA256"] = file_hash + + # Save info file + info_path = os.path.splitext(file_path)[0] + ".info" + with open(info_path, 'w', encoding='utf-8') as f: + json.dump(civitai_data, f, ensure_ascii=True, indent=4) + + except Exception as e: + GLOBAL_SCAN_STATE["error"] = f"{filename}: {e}" + + except Exception as e: + GLOBAL_SCAN_STATE["error"] = str(e) + traceback.print_exc() + finally: + GLOBAL_SCAN_STATE["scanning"] = False + GLOBAL_SCAN_STATE["filename"] = "" + + # Clear caches + if hasattr(folder_paths, "filename_list_cache"): + folder_paths.filename_list_cache.clear() + if hasattr(folder_paths, "cache_helper") and hasattr(folder_paths.cache_helper, "clear"): + folder_paths.cache_helper.clear() + + try: + threading.Thread(target=run_deep_scan, daemon=True).start() + except Exception as e: + GLOBAL_SCAN_STATE["scanning"] = False + GLOBAL_SCAN_STATE["error"] = str(e) + return web.json_response({"status": "error", "message": str(e)}, status=500) + return web.json_response({"status": "success", "message": "Deep scan started in background"}) diff --git a/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/utils.py b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6e5653895a3051d24889827a96ffabe7c95d72a5 --- /dev/null +++ b/ComfyUI/custom_nodes/Anomalous_Model_Browser/api/utils.py @@ -0,0 +1,610 @@ +import os +import sys +import json +import urllib.parse +import urllib.request +import subprocess +import threading +import asyncio +from aiohttp import web +import folder_paths +import struct +import hashlib +import tempfile +import time + + +CARD_THUMBNAIL_EDGE = 512 +CARD_THUMBNAIL_CACHE_LIMIT = 256 * 1024 * 1024 +CARD_THUMBNAIL_STATIC_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.webp', '.avif'} +_thumbnail_cleanup_lock = threading.Lock() +_thumbnail_generation_slots = threading.BoundedSemaphore(2) +_thumbnail_last_cleanup = 0.0 +_thumbnail_created_since_cleanup = 0 + + +def resolve_within(base_dir, *parts): + """Resolve a path and require it to stay inside base_dir, including through symlinks.""" + base_real = os.path.realpath(base_dir) + candidate = os.path.realpath(os.path.join(base_real, *parts)) + try: + if os.path.commonpath([base_real, candidate]) != base_real: + raise ValueError("Path escapes the configured directory") + except ValueError: + raise ValueError("Path escapes the configured directory") + return candidate + + +def get_folder_root(folder_type, path_idx=0): + paths = folder_paths.get_folder_paths(folder_type) + if not paths or not isinstance(path_idx, int) or path_idx < 0 or path_idx >= len(paths): + raise ValueError("Invalid folder path") + return os.path.realpath(paths[path_idx]) + + +def resolve_folder_subdir(folder_type, path_idx=0, subfolder='/'): + base_dir = get_folder_root(folder_type, path_idx) + relative = "" if subfolder in (None, "", "/") else str(subfolder).strip("/\\") + return base_dir, resolve_within(base_dir, relative) + + +def require_filename(filename): + if not isinstance(filename, str) or not filename or os.path.basename(filename) != filename: + raise ValueError("Invalid filename") + return filename + + +def _thumbnail_cache_directory(): + try: + base_dir = folder_paths.get_temp_directory() + except Exception: + base_dir = None + if not isinstance(base_dir, (str, os.PathLike)): + base_dir = tempfile.gettempdir() + cache_dir = os.path.join(base_dir, "anomalous_model_browser", "card_thumbnails") + os.makedirs(cache_dir, exist_ok=True) + return cache_dir + + +def _prune_thumbnail_cache(cache_dir, created=False): + """Occasionally cap the derived thumbnail cache without touching source covers.""" + global _thumbnail_last_cleanup, _thumbnail_created_since_cleanup + now = time.monotonic() + with _thumbnail_cleanup_lock: + if created: + _thumbnail_created_since_cleanup += 1 + now = time.monotonic() + if ( + now - _thumbnail_last_cleanup < 3600 + and _thumbnail_created_since_cleanup < 64 + ): + return + _thumbnail_last_cleanup = now + _thumbnail_created_since_cleanup = 0 + entries = [] + total_size = 0 + try: + with os.scandir(cache_dir) as iterator: + for entry in iterator: + if not entry.is_file() or not entry.name.endswith('.webp'): + continue + try: + stat = entry.stat() + except OSError: + continue + total_size += stat.st_size + entries.append((stat.st_mtime_ns, stat.st_size, entry.path)) + except OSError: + return + if total_size <= CARD_THUMBNAIL_CACHE_LIMIT: + return + target_size = int(CARD_THUMBNAIL_CACHE_LIMIT * 0.8) + for _, size, path in sorted(entries): + try: + os.remove(path) + total_size -= size + except OSError: + pass + if total_size <= target_size: + break + + +def _build_card_thumbnail_impl(source_path): + """Return a cached 512px WebP card image, or the original on any safe fallback.""" + try: + from PIL import Image, ImageOps + + source_stat = os.stat(source_path) + cache_key = "\0".join(( + os.path.realpath(source_path), + str(source_stat.st_size), + str(source_stat.st_mtime_ns), + str(getattr(source_stat, 'st_ctime_ns', 0)), + str(CARD_THUMBNAIL_EDGE), + )) + digest = hashlib.sha256(cache_key.encode('utf-8', errors='surrogatepass')).hexdigest() + cache_dir = _thumbnail_cache_directory() + cached_path = os.path.join(cache_dir, f"{digest}.webp") + if os.path.isfile(cached_path): + try: + os.utime(cached_path, None) + except OSError: + pass + _prune_thumbnail_cache(cache_dir) + return cached_path + + with Image.open(source_path) as image: + if getattr(image, 'is_animated', False): + return source_path + image = ImageOps.exif_transpose(image) + if max(image.size) <= CARD_THUMBNAIL_EDGE: + return source_path + resampling = getattr(Image, 'Resampling', Image).LANCZOS + image.thumbnail((CARD_THUMBNAIL_EDGE, CARD_THUMBNAIL_EDGE), resampling) + if image.mode not in ('RGB', 'RGBA'): + image = image.convert('RGBA' if 'transparency' in image.info else 'RGB') + temp_path = ( + f"{cached_path}.{os.getpid()}.{threading.get_ident()}.tmp" + ) + try: + image.save(temp_path, format='WEBP', quality=84, method=4) + os.replace(temp_path, cached_path) + finally: + if os.path.exists(temp_path): + try: + os.remove(temp_path) + except OSError: + pass + _prune_thumbnail_cache(cache_dir, created=True) + return cached_path + except Exception: + return source_path + + +def _build_card_thumbnail(source_path): + # Bound simultaneous decodes so opening a large folder cannot monopolize CPU/RAM. + with _thumbnail_generation_slots: + return _build_card_thumbnail_impl(source_path) + +async def api_serve_image(request): + """Dedicated image serving endpoint for model preview images.""" + folder_type = request.query.get('type', 'checkpoints') + try: + path_idx = int(request.query.get('path_idx', 0)) + except: + path_idx = 0 + subfolder = request.query.get('subfolder', '') + filename = request.query.get('filename', '') + + try: + filename = require_filename(filename) + _, target_dir = resolve_folder_subdir(folder_type, path_idx, subfolder) + file_path = resolve_within(target_dir, filename) + except (ValueError, KeyError): + return web.Response(status=400, text='Invalid request') + + if not os.path.exists(file_path) or not os.path.isfile(file_path): + return web.Response(status=404, text='Image not found') + + ext = os.path.splitext(filename)[1].lower() + content_types = { + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.webp': 'image/webp', + '.gif': 'image/gif', + '.avif': 'image/avif', + '.mp4': 'video/mp4', + '.webm': 'video/webm', + '.mov': 'video/quicktime', + '.avi': 'video/x-msvideo' + } + content_type = content_types.get(ext) + if content_type is None: + return web.Response(status=415, text='Unsupported media type') + + served_path = file_path + if ( + request.query.get('variant') == 'card' + and ext in CARD_THUMBNAIL_STATIC_EXTENSIONS + ): + served_path = await asyncio.to_thread(_build_card_thumbnail, file_path) + if served_path != file_path: + content_type = 'image/webp' + + headers = {'Content-Type': content_type} + if request.query.get('t'): + headers['Cache-Control'] = 'public, max-age=31536000, immutable' + return web.FileResponse(served_path, headers=headers) + + + +async def api_clear_cache(request): + try: + from .metadata import clear_metadata_cache + + if hasattr(folder_paths, "filename_list_cache"): + folder_paths.filename_list_cache.clear() + if hasattr(folder_paths, "cache_helper") and hasattr(folder_paths.cache_helper, "clear"): + folder_paths.cache_helper.clear() + clear_metadata_cache() + return web.json_response({"status": "success"}) + except Exception as e: + return web.json_response({"status": "error", "message": str(e)}) + + +async def api_translate(request): + try: + data = await request.json() + text = data.get("text", "") + tl = data.get("target_lang", "zh-CN") + if not text: + return web.json_response({"translated": ""}) + + # Try DeepL if API key exists + script_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(script_dir, "config.json") + deepl_key = "" + if os.path.exists(config_path): + try: + with open(config_path, 'r', encoding='utf-8') as f: + cfg = json.load(f) + deepl_key = cfg.get("DEEPL_API_KEY", "").strip() + except: + pass + + if deepl_key: + deepl_map = { "zh-CN": "ZH", "en": "EN", "ja": "JA", "ko": "KO", "fr": "FR", "de": "DE", "es": "ES", "ru": "RU" } + d_tl = deepl_map.get(tl, "EN") + url = "https://api-free.deepl.com/v2/translate" if ":fx" in deepl_key else "https://api.deepl.com/v2/translate" + payload = urllib.parse.urlencode({ + "auth_key": deepl_key, + "text": text, + "target_lang": d_tl + }).encode('utf-8') + req = urllib.request.Request(url, data=payload) + with urllib.request.urlopen(req, timeout=5) as resp: + result = json.loads(resp.read().decode('utf-8')) + return web.json_response({"translated": result["translations"][0]["text"]}) + else: + # Fallback to Google Translate (free, no key) + url = f"https://translate.googleapis.com/translate_a/single?client=gtx&sl=auto&tl={tl}&dt=t&q={urllib.parse.quote(text)}" + req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'}) + with urllib.request.urlopen(req, timeout=5) as resp: + result = json.loads(resp.read().decode('utf-8')) + # Google Translate returns a list of fragments + translated_text = "".join([part[0] for part in result[0]]) + return web.json_response({"translated": translated_text}) + + except Exception as e: + print(f"Translate Error: {e}") + return web.json_response({"translated": text, "error": str(e)}) + + +async def api_get_gallery_images(request): + try: + output_dir = folder_paths.get_output_directory() + if not os.path.exists(output_dir): + return web.json_response({"images": [], "total": 0, "page": 1, "pages": 0}) + + page = max(1, int(request.query.get('page', 1))) + limit = min(200, max(1, int(request.query.get('limit', 50)))) + + valid_exts = {'.png', '.jpg', '.jpeg', '.webp', '.gif'} + def collect_images(): + images = [] + for root, dirs, files in os.walk(output_dir): + for f in files: + ext = os.path.splitext(f)[1].lower() + if ext in valid_exts: + rel_path = os.path.relpath(root, output_dir) + subfolder = "" if rel_path == "." else rel_path.replace('\\', '/') + full_path = os.path.join(root, f) + try: + mtime = os.path.getmtime(full_path) + except OSError: + mtime = 0 + images.append({ + "filename": f, + "subfolder": subfolder, + "type": "output", + "mtime": mtime + }) + images.sort(key=lambda x: x['mtime'], reverse=True) + return images + + images = await asyncio.to_thread(collect_images) + + total = len(images) + start_idx = (page - 1) * limit + end_idx = start_idx + limit + paginated_images = images[start_idx:end_idx] + + return web.json_response({ + "images": paginated_images, + "total": total, + "page": page, + "pages": (total + limit - 1) // limit if limit > 0 else 0 + }) + except Exception as e: + return web.json_response({"error": str(e)}, status=500) + +async def api_delete_gallery_image(request): + try: + data = await request.json() + filename = data.get("filename") + subfolder = data.get("subfolder", "") + + output_dir = folder_paths.get_output_directory() + try: + filename = require_filename(filename) + target_dir = resolve_within(output_dir, subfolder) + file_path = resolve_within(target_dir, filename) + except ValueError: + return web.json_response({"status": "error", "message": "Invalid parameters"}, status=400) + + if os.path.exists(file_path): + os.remove(file_path) + return web.json_response({"status": "success"}) + else: + return web.json_response({"status": "error", "message": "File not found"}) + except Exception as e: + return web.json_response({"status": "error", "message": str(e)}, status=500) + +def read_png_text_fast(path): + try: + with open(path, 'rb') as f: + signature = f.read(8) + if signature != b'\x89PNG\r\n\x1a\n': + return None + while True: + length_bytes = f.read(4) + if not length_bytes: break + length = struct.unpack('>I', length_bytes)[0] + chunk_type = f.read(4) + if chunk_type == b'tEXt': + data = f.read(length) + keyword, text = data.split(b'\0', 1) + if keyword == b'prompt': + return text.decode('utf-8', errors='ignore') + else: + f.seek(length, 1) + f.seek(4, 1) + except Exception: + pass + return None + + +async def api_get_model_images(request): + model_name = request.rel_url.query.get('model_name', '') + if not model_name: + return web.json_response({'images': []}) + + base_target = os.path.basename(model_name).lower() + + # We will search the output directory + output_dir = folder_paths.get_output_directory() + if not os.path.exists(output_dir): + return web.json_response({'images': []}) + + def find_images(): + matched_images = [] + for root, _, files in os.walk(output_dir): + for file in files: + if not file.lower().endswith('.png'): + continue + full_path = os.path.join(root, file) + prompt_text = read_png_text_fast(full_path) + if not prompt_text: + continue + try: + prompt_data = json.loads(prompt_text) + matched = any( + os.path.basename(v).lower() == base_target + for node in prompt_data.values() + if isinstance(node, dict) and 'class_type' in node + for v in node.get('inputs', {}).values() + if isinstance(v, str) + ) + if not matched: + continue + rel_path = os.path.relpath(root, output_dir).replace('\\', '/') + if rel_path == '.': + rel_path = '' + url = f'/view?filename={urllib.parse.quote(file)}&type=output' + if rel_path: + url += f'&subfolder={urllib.parse.quote(rel_path)}' + matched_images.append({'url': url, 'mtime': os.path.getmtime(full_path)}) + except (OSError, ValueError, TypeError, json.JSONDecodeError): + pass + matched_images.sort(key=lambda x: x['mtime'], reverse=True) + return matched_images + + matched_images = await asyncio.to_thread(find_images) + + return web.json_response({'images': matched_images}) + +def get_folder_view_mode(): + script_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(script_dir, "config.json") + try: + if os.path.exists(config_path): + with open(config_path, 'r', encoding='utf-8') as f: + cfg = json.load(f) + return cfg.get("folder_view_mode", "abstract") + except: + pass + return "abstract" + +def get_all_physical_basenames(): + basenames = set() + for t in folder_paths.folder_names_and_paths.keys(): + try: + paths = folder_paths.get_folder_paths(t) + if not paths: continue + for p in paths: + bn = os.path.basename(os.path.normpath(p)) + if bn: basenames.add(bn) + except: + pass + return list(basenames) + +def get_active_physical_basenames(): + script_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(script_dir, "config.json") + + all_bns = get_all_physical_basenames() + active = [] + configured = set() + + try: + if os.path.exists(config_path): + with open(config_path, 'r', encoding='utf-8') as f: + cfg = json.load(f) + pfc = cfg.get("physical_folders_config") + if pfc and isinstance(pfc, list): + for item in pfc: + bn = item.get("type") + if bn in all_bns: + configured.add(bn) + if item.get("visible", True): + active.append(bn) + + for bn in all_bns: + if bn not in configured: + active.append(bn) + return active + except: + pass + + return all_bns + +def get_active_scan_paths(): + mode = get_folder_view_mode() + paths = set() + + if mode == "physical": + active_bns = set(get_active_physical_basenames()) + for t in folder_paths.folder_names_and_paths.keys(): + try: + ps = folder_paths.get_folder_paths(t) + if not ps: continue + for p in ps: + if not os.path.exists(p): continue + bn = os.path.basename(os.path.normpath(p)) + if bn in active_bns: + paths.add(os.path.realpath(p)) + except: + pass + else: + active_types = get_active_folder_types() + for t in active_types: + try: + ps = folder_paths.get_folder_paths(t) + if not ps: continue + for p in ps: + if not os.path.exists(p): continue + paths.add(os.path.realpath(p)) + except: + pass + + return list(paths) + +def get_active_folder_types(): + script_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(script_dir, "config.json") + + all_types = list(folder_paths.folder_names_and_paths.keys()) + default_types = ['checkpoints', 'loras', 'diffusion_models', 'controlnet', 'vae'] + + try: + if os.path.exists(config_path): + with open(config_path, 'r', encoding='utf-8') as f: + cfg = json.load(f) + ftc = cfg.get("folder_types_config") + if ftc and isinstance(ftc, list): + active = [] + configured_types = set() + for item in ftc: + t = item.get("type") + if t in all_types: + configured_types.add(t) + if item.get("visible", True): + active.append(t) + # For any newly registered types not in config, do not show them by default + # unless they are in default_types + for t in all_types: + if t not in configured_types and t in default_types: + active.append(t) + return active + except Exception: + pass + + # Fallback if no config exists + active = [t for t in default_types if t in all_types] + return active + +async def api_get_all_folder_types(request): + mode = get_folder_view_mode() + script_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(script_dir, "config.json") + + result = [] + + if mode == "physical": + all_bns = get_all_physical_basenames() + configured = set() + try: + if os.path.exists(config_path): + with open(config_path, 'r', encoding='utf-8') as f: + cfg = json.load(f) + pfc = cfg.get("physical_folders_config") + if pfc and isinstance(pfc, list): + for item in pfc: + bn = item.get("type") + if bn in all_bns: + configured.add(bn) + result.append({ + "type": bn, + "visible": item.get("visible", True) + }) + except: + pass + + default_physical_types = ['checkpoints', 'loras', 'unet', 'diffusion_models', 'controlnet', 'vae'] + for bn in all_bns: + if bn not in configured: + result.append({ + "type": bn, + "visible": bn in default_physical_types + }) + else: + # Abstract mode + all_types = list(folder_paths.folder_names_and_paths.keys()) + default_types = ['checkpoints', 'loras', 'diffusion_models', 'controlnet', 'vae'] + configured_types = set() + try: + if os.path.exists(config_path): + with open(config_path, 'r', encoding='utf-8') as f: + cfg = json.load(f) + ftc = cfg.get("folder_types_config") + if ftc and isinstance(ftc, list): + for item in ftc: + t = item.get("type") + if t in all_types: + configured_types.add(t) + result.append({ + "type": t, + "visible": item.get("visible", True) + }) + except: + pass + + for t in all_types: + if t not in configured_types: + result.append({ + "type": t, + "visible": t in default_types + }) + + return web.json_response({"folder_types": result, "folder_view_mode": mode}) diff --git a/ComfyUI/custom_nodes/Anomalous_Model_Browser/config.example.json b/ComfyUI/custom_nodes/Anomalous_Model_Browser/config.example.json new file mode 100644 index 0000000000000000000000000000000000000000..d0acb619aa3cdf28d5dc9fc60b9ed7bb59626f9b --- /dev/null +++ b/ComfyUI/custom_nodes/Anomalous_Model_Browser/config.example.json @@ -0,0 +1,3 @@ +{ + "CIVITAI_API_KEY": "" +} \ No newline at end of file diff --git a/ComfyUI/custom_nodes/Anomalous_Model_Browser/error_and_experience_summary.md b/ComfyUI/custom_nodes/Anomalous_Model_Browser/error_and_experience_summary.md new file mode 100644 index 0000000000000000000000000000000000000000..340f662ebbb3d20572ae793af6c2d876fd937d4c --- /dev/null +++ b/ComfyUI/custom_nodes/Anomalous_Model_Browser/error_and_experience_summary.md @@ -0,0 +1,293 @@ +# 📖 Error & Experience Summary (开发避坑与经验总结) + +This document serves as an architectural retrospective and UX diagnostic log for the development of the Anomalous Model Browser. +这份文档旨在记录 Anomalous 插件开发过程中的架构演进、UX 设计教训以及核心 Bug 排查经验。 + +--- + +## 1. UX Scroll Traps (交互嵌套滚动陷阱) +**The Problem (问题)**: In the Notebook modal, the translation editor had a `max-height` and `overflow-y`. Since the parent modal was also scrollable, it created a "Scroll Trap". Hitting the bottom of the editor unpredictably transferred the scroll event to the outer body. +(在笔记本弹窗中,由于翻译编辑器固定了最大高度并允许滚动,导致与父级弹窗的滚动条发生冲突。用户滚动到底部时会突然把外层页面卷走,体验极差。) +**The Solution (方案)**: Removed the `max-height` and internal scrollbar. Letting the translation editor expand naturally utilizing the primary window scrollbar guarantees a continuous experience. +(彻底移除内部最大高度和滚动条。让编辑器自然撑开,统一使用最外层的主滚动条,保证心流不断档。) + +## 2. Artificial Metadata Pollution (伪造数据的反噬) +**The Problem (问题)**: To make the "Base Model" dropdown look complete, the backend artificially injected hardcoded architectures (`SDXL`, `HunyuanVideo`) even if the user didn't own them. This confused users. +(为了让“模型架构”下拉菜单看起来更“丰满”,后端硬编码注入了一些用户根本没下载过的新架构。结果让用户极度困惑,以为自己眼花了。) +**The Solution (方案)**: Data truth must strictly reflect the local disk state. Dropped artificial padding; the UI dropdown now accurately mirrors only the physical files on disk. +(数据必须绝对忠于本地磁盘。删除了所有的硬编码注入,确保下拉菜单严格反映本地真实的生态。) + +## 3. UI Strobe Effects on Grid Hovers (网格悬浮的“光污染”) +**The Problem (问题)**: Standard `transition: all 0.2s` meant quickly sweeping the mouse across the model grid caused all cards to rapidly light up like a strobe light. +(为卡片设置了标准的 0.2秒 悬浮动画,导致鼠标快速扫过网格时,所有卡片像迪厅频闪一样疯狂亮起,显得非常廉价。) +**The Solution (方案)**: Introduced a `transition-delay: 0.08s` low-pass filter. The recommended UX range for hover intent is 50ms - 100ms. This 80ms delay is imperceptible for intentional clicks but completely eliminates flashes from accidental sweeps. +(引入了 `0.08s` 动画延迟作为低通滤波器。根据人机交互体验,区分“无意滑过”与“有意悬停”的黄金推荐区间是 50ms - 100ms。我们采用的 80ms 延迟对正常点击毫无察觉,但完美过滤了鼠标无意扫过造成的闪烁。) + +## 4. The ComfyUI Manager Auto-Update File Wipe (Manager自动更新导致文件丢失) +**The Problem (问题)**: A custom documentation file (`docs/核心按钮操作指南_大白话版.md`) generated by AI locally was silently deleted when clicking "Update" on the node inside ComfyUI Manager. +(本地让 AI 写的说明文件,在点击 ComfyUI Manager 的更新按钮后离奇失踪了。) +**The Solution (方案)**: ComfyUI Manager's update mechanism often performs a raw `git pull` or hard reset. If a local file is not tracked in the remote GitHub repository (i.e., you haven't committed and pushed it), it is highly likely to be overwritten or wiped during the sync process. Always commit and push your documentation, or keep personal untracked notes outside the extension directory! +(ComfyUI Manager 的更新机制依赖 Git 同步(类似 `git reset --hard` 或强制 pull)。如果你在本地创建了文件却没有提交并推送到 GitHub,更新时 Manager 会强行把本地状态与云端对齐,导致这些“未追踪(untracked)”的文件被直接抹杀。教训:重要的文档一定要及时 push,或者别放在插件目录下!) + +## 5. "Hostage" Inline Confirmations (被“绑架”的内联确认) +**The Problem (问题)**: Clicking "Delete" changed the button to "Are you sure?" with a 3-second timeout, but lacked a "Cancel" button. Users misclicking were held hostage for 3 seconds. +(点击删除后会变成“确定删除吗?”并锁定 3 秒,但没有“取消”按钮。用户如果不小心点错,只能像被绑架一样干等 3 秒它恢复。) +**The Solution (方案)**: Wrapped the logic to instantly reveal a secondary `[✕]` cancel button upon the first click, giving control back immediately. +(在点击删除时,立马在旁边弹出 `[✕]` 取消按钮。点击取消直接清空倒计时,立刻把控制权还给用户。) + +## 6. First-load "Dead Interface" Avoidance (首次加载的“白屏假死”) +**The Problem (问题)**: Initializing the sidebar to the root directory `/` presented a blank screen if users only had models nested deep in subfolders. +(插件默认打开根目录 `/`。如果用户的模型全都在很深的子文件夹里,打开插件就会看到一片空白,以为插件坏了。) +**The Solution (方案)**: Implemented `firstLoadDone` to automatically force the UI to open the very first directory possessing `model_count > 0`. +(加入了智能寻路逻辑,插件首次加载时自动扫描目录树,直接定位并打开第一个存有模型的文件夹。) + +## 7. Structural Layout Breakages (流式布局的坍塌) +**The Problem (问题)**: Placing varying lengths of translation tags sequentially caused extreme layout jitter when hovering. +(把中英文标签简单并排放在一起,由于字符串长度不一,鼠标悬浮时极容易发生换行挤压,导致整个界面疯狂跳动。) +**The Solution (方案)**: Enforced a strict "Row-based" flex architecture (`align-items: stretch`). Each tag pair is locked into a geometrical row. +(采用极其严格的行级 Flex 布局,把中英文双语硬性锁死在同一个拉伸行内,无论字数多少,几何结构绝对稳定。) + +## 8. Modal State Persistence (弹窗状态的幽灵) +**The Problem (问题)**: Closing and reopening the Notebook modal showed a blank editor because the JS object cached `this.currentNotebook` but the DOM was obliterated. +(关掉笔记本弹窗再打开,右侧编辑器全空。因为 JS 内存里还记着上一次的状态,但 DOM 节点已经被销毁重建了。) +**The Solution (方案)**: Explicitly check cached state during modal initialization and force `renderNotebookEditor()` if necessary. +(在弹窗初始化时主动校验缓存,强制触发重新渲染,实现了完美的“会话级记忆”。) + +## 9. Missing Visual Feedback for Background Operations (无反馈的幽灵进程) +**The Problem (问题)**: The Notebook "Save" button executed a background `fetch` perfectly, but provided absolutely no visual feedback. Users assumed it was broken. +(点击“保存”按钮后,后台默默完成了请求,但按钮本身没有任何变化。用户以为没点上,引发数据丢失的焦虑。) +**The Solution (方案)**: Clicking Save instantly changes the button to a green `✅` state for 1.5 seconds. +(点击保存后强制变绿 `✅` 1.5 秒,给用户极强的心理安全感。) + +## 10. Spatial Geometry for Multi-Node Spawning (多节点生成的空间重叠) +**The Problem (问题)**: Spawning an entire Notebook (Main Model + Loras + CLIPs) caused all nodes to clump perfectly at `(0,0)`. +(一键生成整个笔记本(主模型+N个Lora+提示词)到画布时,所有节点全叠在同一个坐标上,变成了一坨。) +**The Solution (方案)**: Implemented a "Relative Offset Matrix Layout" (`X + 350*N`, `Y + 250`) and bound it to the `mousemove` event (Magnetic Sticking). +(加入了偏移矩阵计算,并绑定鼠标移动。用户可以直观地拖着排成流水的节点序列在画布上移动,然后再点击放下。) + +## 11. Localization Host-Environment Constraints (寄生环境的本地化陷阱) +**The Problem (问题)**: Reading `document.documentElement.lang` failed because ComfyUI forces the host HTML to `en-US`, breaking our Chinese translations. +(插件试图读取网页根节点的 lang 属性来决定语言。但 ComfyUI 强行把底座设成了纯英文,导致中国用户永远看不到中文提示。) +**The Solution (方案)**: Never trust the host document properties. Rerouted localization strictly to our internal `currentLang` variable. +(永远不要相信宿主环境。彻底切断外部依赖,纯靠插件内部自己的 `currentLang` 变量来统治中英文环境。) + +## 12. Strict Path Separator Validation (路径分隔符的玄学崩溃) +**The Problem (问题)**: Windows uses backslashes (`\`), but assigning `anime/model` to a node's combo widget failed ComfyUI's strict string matching, causing red nodes. +(Windows 的底层模型路径带反斜杠,但前端组件要求精确匹配字符串,哪怕是一个斜杠方向不对,节点也会标红报错。) +**The Solution (方案)**: Built `setWidgetValuePath` to normalize both target paths and native options with `.replace(/\\/g, '/')` before assignment. +(做了一个拦截器,强行把系统内的选项和我们要赋的值全部统一转成正斜杠再进行比较,直接绕过操作系统的路径陷阱。) + +## 13. Backend Starvation During Inference (算力挤兑导致的接口饿死) +**The Problem (问题)**: Fetching plugin data while ComfyUI is generating an image causes timeouts because of PyTorch GIL-locking. +(在 ComfyUI 画图满载的时候,点插件会导致接口超时。因为单线程的 Python 完全被 GPU 推理抢占了。) +**The Solution (方案)**: Accept it gracefully. Database management and intense GPU inference are mutually exclusive workflows. +(优雅地接受这个现实。不需要过度工程化地去做离线队列,画图和管理模型本来就不该同时进行。) + +## 14. DOM Obliteration vs Virtual DOM Hoarding (物理毁灭 vs 虚拟回收) +**The Problem (问题)**: Maintaining performance when navigating folders containing thousands of models. +(当一个文件夹里有上千个模型时,频繁切换文件夹会导致浏览器内存爆炸。) +**The Solution (方案)**: Instead of `display: none` hoarding, strictly enforce `this.grid.innerHTML = ''` to physically obliterate nodes and trigger immediate Garbage Collection. +(放弃“隐藏重用”的传统思路。切文件夹时直接暴力清空内部的 HTML 节点,强迫浏览器立马回收内存,保持零负担。) + +## 15. The "Missing Plugin Icon" Trap (LocalStorage 坐标越界飞天) +**The Problem (问题)**: If the floating trigger button was saved at `X=3000px` on a 4K monitor, loading it on a `1920px` laptop rendered it permanently off-screen. +(在大显示器上把插件按钮拖到了最右边,换到小笔记本上时,坐标超出了屏幕,按钮彻底消失,再也点不出来了。) +**The Solution (方案)**: Implemented a bounds clamp checking coordinates against `window.innerWidth`. If out of bounds, snap it to the bottom-right corner. +(在启动时强制对比当前屏幕大小,一旦发现越界,立刻把它抓回到右下角的安全区内。) + +## 16. The `[vite:preloadError]` Template Literal Crash (一个反引号引发的血案) +**The Problem (问题)**: A missing backtick in a dynamic ES Module dictionary crashed the entire ComfyUI extension loader before execution. +(在汉化字典里不小心删掉了一个反引号,导致 Vite 打包工具报语法错误,整个插件直接在加载前暴毙,UI 按钮全都没了。) +**The Solution (方案)**: JS zero-tolerance for syntax errors requires extreme validation. Fixed the missing backtick. +(原生 JS 没有任何编译容错,任何语法拼写都必须如履薄冰,及时修复了那个反引号。) + +## 17. Target Size Deduplication Bug (重叠目录的体积匹配悖论) +**The Problem (问题)**: Fixing models by file size required finding *exactly one* matching file. But users mapping `checkpoints` and `diffusion_models` to the same folder caused `os.walk` to find the same physical file twice, aborting the fix. +(物理寻址兜底算法要求“体积必须唯一”。但用户配置了多个虚拟路径指向同一个物理文件夹,导致同一个文件被扫出两次,系统误判为“体积不唯一”而放弃修复。) +**The Solution (方案)**: Added `os.path.realpath` deduplication matrix to condense multiple virtual paths back to a single hard drive sector. +(引入了绝对物理地址 `realpath` 去重机制,无论虚拟路径怎么重叠,物理层面上都将其归一化。) + +## 18. The "Lightweight Scan" Re-Hash Catastrophe (轻量扫描的重算灾难) +**The Problem (问题)**: The lightweight scan (`--skip-media`) demanded *both* an `.info` file and a preview image to skip hashing. Since it didn't download images, it forcefully re-hashed hundreds of GBs of models every time. +(轻量化极速扫描因为不会下载图片,导致“是否跳过计算”的判断条件(既有 info 又有图)永远不成立。引发了灾难性的全盘重新计算 SHA256,慢如蜗牛。) +**The Solution (方案)**: Modified skip logic conditionally: if explicitly skipping media, the existence of `.info` is sufficient to bypass hashing. +(加入了情境判断:只要是在极速模式下,不管有没有图,只要有文字信息就无条件跳过,恢复了真正的极速。) + +## 19. The Stale Dropdown Visual Residue (顽固的红框残留) +**The Problem (问题)**: Even after a model path was auto-fixed, ComfyUI maintained a permanent red error border on the node if the frontend cache was stale. +(底层其实已经修复并加载了正确的模型,但因为前端选项缓存没刷过来,ComfyUI 死死把节点标红,看着十分闹心。) +**The Solution (方案)**: Forcefully execute `delete node.color; delete node.bgcolor;` to violently strip native error states without requiring a refresh. +(既然修好了,就直接暴力删除节点的 `color` 属性,强行洗白红框,无需刷新网页。) + +## 20. The Silent Subprocess Death (DEVNULL 黑洞) +**The Problem (问题)**: `scraper.py` crashed due to Python indentation, but piping `stderr` to `DEVNULL` made it totally invisible. The UI waited forever. +(后台脚本缩进写错了导致崩溃,但代码里把报错信息塞进了无尽深渊 `DEVNULL`,没有任何提示,前端界面只能干等着,陷入死局。) +**The Solution (方案)**: Never pipe `stderr` to DEVNULL during active development. Let it log. +(开发阶段绝对不要屏蔽底层报错,必须让 `stderr` 打印到控制台以便排查。) + +## 21. Tiered Fallback Resolution for O(1) Scans (多级降级寻址打造 O(1) 扫描) +**The Problem (问题)**: Unscanned 10GB models required full disk read to calculate SHA256, taking minutes. +(即便是第一次扫描,遇到几十 GB 的新模型,完整计算 SHA256 也要几分钟,用户体验极度卡顿。) +**The Solution (方案)**: +1. 0ms: Read existing `.info`. (直接读取已有的 info) +2. 0.01s: Extract 64KB JSON header from `.safetensors` via `struct` for pre-computed hash. (用 struct 撬开文件头瞬间提取内置的 Hash,降维打击) +3. Fallback: Full SHA256 only for legacy models. (真找不到了才迫不得已做全量计算) + +## 22. Zero-API Tensor Fingerprinting (零API脱机张量基因识别) +**The Problem (问题)**: HuggingFace or private models returned 404 from Civitai, failing to generate `.info` and completely breaking UI integration. +(从 HF 拿到的或者是自己练的私模,由于 C站 没有记录,API 直接报错 404,导致模型彻底被忽略,前端里连看都看不到。) +**The Solution (方案)**: Read the first 500 Tensor Keys via struct. Identifying `double_blocks.0.img_attn` strictly confirms Flux.1 D architecture completely offline. +(不管你有没有网,强行读取模型前 500 层的网络结构。只要看到里面有 `double_blocks`,直接盖章确认为 Flux 架构。完全脱机,无缝融入前端兼容雷达。) + +## 23. Gemini-Style Popover UI (告别全屏遮罩,采用悬浮弹窗) +**The Problem (问题)**: Center-screen settings modals with blur backgrounds felt heavy and disconnected users from the workspace. +(原本的设置面板是个带有大面积模糊遮罩的全屏居中弹窗,显得特别笨重,像个垃圾网页游戏。) +**The Solution (方案)**: Refactored to a lightweight side-popover (`left: 100%; bottom: 15px;`) imitating Gemini's web UI, maintaining deep workflow immersion. +(改成绝对定位,像 Gemini 网页端那样从侧边栏右侧悄悄弹出一个悬浮气泡,点外侧自动收起,极大保证了沉浸感。) + +## 24. Dock & Mode Lock (全宽底座与多状态锁死) +**The Problem (问题)**: Floating capsules looked cramped, and entering fullscreen Gallery while the Sidebar menu remained clickable caused DOM rendering conflicts. +(底部按钮挤在一起极其难看。而且进入全屏图库的时候,居然还能点左上角的汉堡菜单,导致页面状态直接崩溃交织在一起。) +**The Solution (方案)**: Extended to a 100% width solid Dock. Disabled sidebar triggers during fullscreen views via strict State Locks. +(把底部做成了满宽的实体底座,稳固了视觉重心。一旦进入图库这种排他性的全视图,强行给菜单上互斥锁,彻底断绝乱按引发的 Bug。) + +## 25. Resolving "Missing" vs "Unscanned" Conflicts (防重叠逻辑拦截自相踩踏) +**The Problem (问题)**: Deleting a model and reloading the workflow popped BOTH the "Missing Model" dark modal and the "Unscanned Model" orange toast. +(把一个模型删掉后重新加载图片,系统一方面觉得“模型丢了”弹出黑框,一方面因为缓存里没它又觉得它是“没扫过的新模型”弹出橙黄色的警告,两个弹窗叠在一块极其弱智。) +**The Solution (方案)**: The unscanned detector now ignores nodes already flagged with red error colors (`node.color === "#FF3333"`), preventing system overlapping. +(加入了双轨防踩踏:未扫描检测器只要看到这个节点已经红了(彻底丢了),就主动闭嘴放弃拦截,边界极其清晰。) + +## 26. The CSS `zoom` Devastation (Zoom 属性对坐标系的降维打击) +**The Problem (问题)**: Using `zoom` to scale the UI broke bounding boxes. `container.offsetHeight > window.innerHeight` caused `Y` coordinates to become negative, permanently launching the window off-screen into the void. +(早期为了偷懒,用 CSS `zoom` 属性来做 UI 缩放。结果它彻底破坏了浏览器的物理坐标系,一旦窗口过大,防越界系统就算出了个负数坐标,导致插件整个闪现飞出屏幕外,再也拖不回来了。) +**The Solution (方案)**: Banned `zoom`. Transitioned to relative scaling via `font-size: calc(16px * var(--anomalous-scale))`. Fixed bounding clamps to execute `< 0` checks last. +(彻底封杀 zoom。改用 `calc(16px * scale)` 的底层字体相对推算放大法。并且把防负数的保底检测代码调到了逻辑的最后一行,永远杜绝负数坐标。) + +## 27. Vanilla JS "Fake" i18n Failure (原生状态管理漏斗下的假性汉化失效) +**The Problem (问题)**: Newly added permanent buttons ("UI Scale", "Reset") remained in Chinese when English was toggled. Since we lacked React/Vue two-way binding, they missed the global state update and got stuck. +(新加了“UI缩放”等按钮后,即使点了切换英文,它们还是中文。因为我们没有使用 Vue 的数据绑定,纯靠原生 JS。如果新加了常驻按钮,却忘了把它登记到顶层 `langBtn.onclick` 突变循环里,它就永远被隔离在旧时间线里了。) +(只要是常驻的按钮,必须强制在语言切换中枢里注册重写一遍。同时开发过程中如果遇到“代码没变”的灵异现象,第一反应永远是按 Ctrl+F5 强制破除浏览器静态缓存。) + +## 28. The Image Cropping Dilemma (图库的裁切与留白博弈) +**The Problem (问题)**: Using `object-fit: cover` in the image gallery created perfectly uniform grid squares, but brutally decapitated tall images (like 9:16 vertical character portraits). +(在图库排版时,为了让网格看起来绝对整齐,使用了 `object-fit: cover`。结果导致偏长的图片(如 9:16 的人物竖图)顶部和底部被无情裁掉,经常出现“砍头”现象。) +**The Solution (方案)**: Changed to `object-fit: contain` and paired it with a deep black background (`#000`). The images now scale down to fit entirely, and the resulting letterboxing seamlessly blends into the dark UI as a natural frame. +(改为 `object-fit: contain`,完整显示图片。由于底层卡片背景本身就是极黑(`#000`),缩小后产生的黑边完美融入了 UI 背景,不仅没有任何违和感,反而像是一圈高级的画框,一举两得。) + +## 29. Vanilla JS Global Click-Outside (原生悬浮面板的全局点击收起) +**The Problem (问题)**: The settings popover only closed if the user explicitly clicked the gear icon again. Clicking elsewhere on the canvas did nothing, making the popover feel rigid and intrusive. +(点击齿轮打开设置悬浮气泡后,必须再点一次齿轮才能关掉。如果随手点一下旁边或者点画布,面板居然不收回,交互显得非常死板和占眼。) +**The Solution (方案)**: Attached a temporary `mousedown` event listener to the `document` when the popover opens. If the click target is outside the modal, it triggers `display: none` and instantly `removeEventListener` to prevent memory leaks. +(面板弹出时,立刻给全局 `document` 挂载一个监听器。只要检测到鼠标点击了面板外部区域,立刻收缩面板,并且**顺手注销掉这个监听器**,实现了极度顺滑的 Gemini 式收放体验且毫无性能负担。) + +## 30. Context Preservation vs UX Intention (上下文保留与用户意图的冲突) +**The Problem (问题)**: After picking a new cover from the Gallery while inside the Model Edit Modal, the UI destroyed the modal, showed the gallery, and then *restored* the modal upon completion to 'preserve context'. However, users who just wanted to change the cover felt confused, thinking they still needed to click 'Save' to apply it. +(在编辑面板中点击更换封面后,UI 会先销毁面板,进入图库,选完后再把编辑面板弹回来以“保留上下文”。但用户其实只想换个封面,重新弹出的面板反而让他们误以为必须再点一次“保存”才算数。) +**The Solution (方案)**: Respect the primary intention. When the cover is changed, directly return to the model card/grid. Do not stubbornly restore intermediate modals unless necessary. +(顺应主要意图。换完封面直接回到网格卡片,干脆利落。不要去死板地恢复中间态面板。) + +## 31. The Missing Reference Crash (引用丢失导致的状态黑洞) +**The Problem (问题)**: A refactor failed to assign `this.models = data.models` in the load function. When a UI callback later tried to find the updated model using `this.models.find()`, JS threw an `undefined` error and aborted execution midway. This left the UI with both the gallery and grid hidden, resulting in a completely blank screen. +(重构时忘了一行赋值代码,导致 `this.models` 为空。后续 UI 切换时试图从中查找模型直接报错崩溃。由于崩溃发生在显示/隐藏面板的切换中途,导致所有面板都被隐藏,屏幕变成了彻底的“真空空白”。) +**The Solution (方案)**: Always ensure foundational state variables are populated before executing callbacks that depend on them. +(底层状态变量必须绝对可靠,一旦中途崩溃,UI 的半途状态是毁灭性的。) + +## 32. Backup Entanglement of Non-Platform Models (非平台模型的备份污染) +**The Problem (问题)**: To ensure a 'Reset' always had a fallback, changing any cover backed up the original file to `.civitai_bak`. For non-Civitai models, this meant user's manual `.png` covers were forcefully renamed to `.civitai_bak`, causing severe semantic confusion. +(为了保证“重置”功能永远有退路,之前的逻辑会把任何被替换的封面强行改名为 `.civitai_bak`。这就导致用户自己放的非 C 站模型封面,也被莫名其妙打上了 C 站的烙印,逻辑极其混乱。) +**The Solution (方案)**: Decouple custom covers by strictly saving them as `.preview.*` and leaving original standard `.png` files untouched. Resetting now just deletes the `.preview.*` file, naturally falling back to the untouched original, or copying the true `.civitai_bak` if it actually came from Civitai. +(让自定义封面独占 `.preview.*` 后缀,绝不触碰原有的标准封面。重置时只需删掉自定义后缀,系统就会自然回退。彻底切断了非 C 站模型与 C 站备份标识的无理纠缠。) + + +## 33. DOM Video Audio Leaks on innerHTML Clear (DOM 销毁导致的幽灵音频泄漏) +**The Problem (问题)**: When navigating from a model detail view containing an actively playing