Upload 24 files
Browse files- custom_nodes/ComfyUI-H3-Motion-Context-main/.comfyignore +9 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/.gitignore +2 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/CHANGELOG.md +171 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/LICENSE +234 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/README.md +477 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/__init__.py +48 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/__pycache__/__init__.cpython-313.pyc +0 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/__pycache__/csrf_guard.cpython-313.pyc +0 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/__pycache__/layout_contract.cpython-313.pyc +0 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/__pycache__/nodes.cpython-313.pyc +0 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/__pycache__/probe_node.cpython-313.pyc +0 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/csrf_guard.py +82 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/example_workflows/MiniMax H3 - fl2va - ref2va.json +0 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/layout_contract.py +243 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/nodes.py +1201 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/probe_node.py +446 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/pyproject.toml +31 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/tests/_mock_harness.py +373 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/tests/_node_smoke_test.py +676 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/tests/_probe_node_test.py +356 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/tests/freeze_detect.py +286 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/tests/level_step.py +334 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/tests/seam_probe.py +262 -0
- custom_nodes/ComfyUI-H3-Motion-Context-main/web/h3_motion_context.js +389 -0
custom_nodes/ComfyUI-H3-Motion-Context-main/.comfyignore
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Caches and local env only. tests/ ships with the pack (smoke test,
|
| 2 |
+
# seam/level/freeze probes, mock harness). Do not blanket-ignore it.
|
| 3 |
+
__pycache__/
|
| 4 |
+
*.py[cod]
|
| 5 |
+
.venv/
|
| 6 |
+
venv/
|
| 7 |
+
.git/
|
| 8 |
+
.github/
|
| 9 |
+
.cursor/
|
custom_nodes/ComfyUI-H3-Motion-Context-main/.gitignore
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.pyc
|
custom_nodes/ComfyUI-H3-Motion-Context-main/CHANGELOG.md
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Changelog
|
| 2 |
+
|
| 3 |
+
Newest first. Dates are release dates.
|
| 4 |
+
|
| 5 |
+
ComfyUI's H3 layout changed shape once so far: `PackedLayout.__init__`
|
| 6 |
+
dropped its `frame_count` parameter along with the restriction that
|
| 7 |
+
rejected any keyframe anchor other than the first or last frame. That
|
| 8 |
+
landed in ComfyUI 0.34.0. Every release through 0.33.4 has the older
|
| 9 |
+
layout. Each entry below says which of the two it works with.
|
| 10 |
+
|
| 11 |
+
## 0.6.2 - 2026-09-06
|
| 12 |
+
|
| 13 |
+
Requires ComfyUI 0.34.0 or newer. Use 0.3.1 on anything older.
|
| 14 |
+
|
| 15 |
+
- Chain `segments` treats a blank value as 0. Graphs saved when the
|
| 16 |
+
button row was the only widget stored `""` there; after `segments` was
|
| 17 |
+
added that empty string failed INT validation.
|
| 18 |
+
- Example workflow: both Chain nodes (fl2va and ref2va) now store
|
| 19 |
+
`segments` 0 instead of that leftover blank.
|
| 20 |
+
|
| 21 |
+
## 0.6.1 - 2026-09-05
|
| 22 |
+
|
| 23 |
+
Requires ComfyUI 0.34.0 or newer. Use 0.3.1 on anything older.
|
| 24 |
+
|
| 25 |
+
- The mock harness no longer uses `importlib.import_module` (that YARA
|
| 26 |
+
hit flagged 0.5.x/0.6.0). `tests/` still ships, including the smoke
|
| 27 |
+
test, seam probe, freeze detect, and level step.
|
| 28 |
+
- Clear latents and slot-exists POST routes require Origin to match Host
|
| 29 |
+
(CSRF). Latent paths must stay inside ComfyUI's output folder.
|
| 30 |
+
|
| 31 |
+
## 0.6.0 - 2026-09-03
|
| 32 |
+
|
| 33 |
+
Requires ComfyUI 0.34.0 or newer. Use 0.3.1 on anything older.
|
| 34 |
+
|
| 35 |
+
- Chain is Approve on a loop: it advances Load/Save, then queues, then
|
| 36 |
+
keeps going. At Load 0 / Save 1 with no clip 1 on disk it generates
|
| 37 |
+
that first clip instead of walking into a missing file. `segments` is
|
| 38 |
+
how many clips that loop runs (0 = until Stop). Reset sets 0/1 and
|
| 39 |
+
does not delete files. Clear latents deletes numbered chain slots;
|
| 40 |
+
custom filenames are left alone.
|
| 41 |
+
|
| 42 |
+
## 0.5.1 - 2026-09-02
|
| 43 |
+
|
| 44 |
+
Requires ComfyUI 0.34.0 or newer. Use 0.3.1 on anything older.
|
| 45 |
+
|
| 46 |
+
- Example workflow updated.
|
| 47 |
+
|
| 48 |
+
## 0.5.0 - 2026-09-02
|
| 49 |
+
|
| 50 |
+
Requires ComfyUI 0.34.0 or newer. Use 0.3.1 on anything older.
|
| 51 |
+
|
| 52 |
+
#25 by feigo313 is why Load 0 is first-clip/no-context, so Motion Context
|
| 53 |
+
stays enabled from the first clip, and why the Chain node exists: walking
|
| 54 |
+
those indices with run-on-change queues twice and skips slots.
|
| 55 |
+
|
| 56 |
+
- Load Latent `clip_index` 0 is first-clip/no-context: nothing is read,
|
| 57 |
+
Motion Context passes the original conditioning through, and
|
| 58 |
+
`trim_frames` is 0. Leave the node enabled. The chain is Load 0 / Save 1,
|
| 59 |
+
then Load 1 / Save 2. Positive indices still load that exact slot.
|
| 60 |
+
The old load-newest meaning of index 0 is gone.
|
| 61 |
+
- H3 Motion Context Chain node: Approve advances Load/Save then queues
|
| 62 |
+
once, Run/Re-roll repeats the current slot (use it instead of
|
| 63 |
+
ComfyUI's Run), Chain auto-approves from the current indices, Reset
|
| 64 |
+
sets 0/1 without queuing. Load, Save, and Chain must share one canvas
|
| 65 |
+
group or the buttons do nothing. Button clicks do not open the node
|
| 66 |
+
menu. Do not use queue "run on change" to walk indices.
|
| 67 |
+
- Save Latent can overwrite a slot that Load still has memory-mapped on
|
| 68 |
+
Windows (os error 1224). Load copies tensors off the file first, and
|
| 69 |
+
Save writes through a temp file.
|
| 70 |
+
|
| 71 |
+
## 0.4.0 - 2026-08-26
|
| 72 |
+
|
| 73 |
+
Requires ComfyUI 0.34.0 or newer. Use 0.3.1 on anything older.
|
| 74 |
+
|
| 75 |
+
ComfyUI now places keyframe anchors at any frame itself, so the two
|
| 76 |
+
runtime patches this pack carried are gone. Nothing in ComfyUI is
|
| 77 |
+
modified any more.
|
| 78 |
+
|
| 79 |
+
- `patch_layout.py` and `patch_payload.py` removed. The node builds plain
|
| 80 |
+
keyframe dicts and hands them to stock code.
|
| 81 |
+
- `layout_contract.py` added. It proves, once before the first render,
|
| 82 |
+
that anchors sit where the arithmetic says and that the pinned audio
|
| 83 |
+
window is placed literally from a fractional, negative anchor index.
|
| 84 |
+
No stock node can produce such an index, so nothing upstream tests it,
|
| 85 |
+
and an integer cast added later would move the pinned sound silently.
|
| 86 |
+
If a check fails the node refuses and names what moved.
|
| 87 |
+
- The pinned audio is a keyframe rather than a reference block. A Ref2VA
|
| 88 |
+
graph's reference list is left untouched.
|
| 89 |
+
- Add Guide for MiniMax H3 anchors survive alongside a chained head,
|
| 90 |
+
including ones carrying their own audio. Guides landing inside the
|
| 91 |
+
pinned head are dropped with a warning, as `first_frame` anchors are.
|
| 92 |
+
- Other packs may patch the layout without this one standing down, since
|
| 93 |
+
it no longer competes for that code. If it finds the constructor
|
| 94 |
+
wrapped it says who by and checks the behaviour anyway.
|
| 95 |
+
- Anchor and audio window coordinates are identical to 0.3.1, verified
|
| 96 |
+
against the real upstream layout on both shapes.
|
| 97 |
+
|
| 98 |
+
## 0.3.1 - 2026-08-14
|
| 99 |
+
|
| 100 |
+
Works with both H3 layouts.
|
| 101 |
+
|
| 102 |
+
Fixed the crash on the newer layout, reported in #12 by javawock7618 and
|
| 103 |
+
#8 by azra1l. The patch had passed `frame_count` unconditionally, so its
|
| 104 |
+
self-test raised `TypeError` and the node refused to run.
|
| 105 |
+
|
| 106 |
+
- The patch reads the layout constructor's signature and adapts, rather
|
| 107 |
+
than assuming either shape. Verified against the real upstream file on
|
| 108 |
+
both: identical anchor and audio window coordinates to the last bit.
|
| 109 |
+
- Keyframe audio latents are no longer dropped from the payload. The
|
| 110 |
+
newer layout lets a keyframe carry audio of its own, and rebuilding the
|
| 111 |
+
list from references alone filled every audio conditioning row with the
|
| 112 |
+
wrong content.
|
| 113 |
+
- Pinned anchors pair with the correct rows when a stock anchor that
|
| 114 |
+
carries audio but no picture shares the graph.
|
| 115 |
+
- The mixed keyframe guard is retired on the newer layout, where stock
|
| 116 |
+
compensates untagged keyframes for reference blocks itself. A Ref2VA
|
| 117 |
+
graph carrying a stock anchor is now supported rather than refused.
|
| 118 |
+
- The test harness fakes both layout shapes.
|
| 119 |
+
|
| 120 |
+
## 0.3.0 - 2026-08-11
|
| 121 |
+
|
| 122 |
+
Works with the older H3 layout only. Refuses to run on the newer one.
|
| 123 |
+
|
| 124 |
+
- **Seam Probe node.** Measures join quality inline in the graph: lag,
|
| 125 |
+
correlation, RMS step and floor level step across the join. Passes
|
| 126 |
+
clip B through unchanged so it can sit inline without rewiring.
|
| 127 |
+
- **last_frame passthrough.** An fl2va graph's own anchors used to be
|
| 128 |
+
replaced outright. A last-frame anchor now survives a chained head,
|
| 129 |
+
tagged so it gets the same reference compensation as the pinned run.
|
| 130 |
+
Anchors falling inside the pinned head are dropped with a warning,
|
| 131 |
+
since the pinned run already decides those frames.
|
| 132 |
+
- Conditioning carrying keyframes resolved against a different clip
|
| 133 |
+
length is refused rather than rendered at the wrong frame.
|
| 134 |
+
|
| 135 |
+
## 0.2.0 - 2026-08-09
|
| 136 |
+
|
| 137 |
+
Works with the older H3 layout only. First release published to the
|
| 138 |
+
ComfyUI registry.
|
| 139 |
+
|
| 140 |
+
- **Reference mode support.** `patch_layout.py` was rewritten to locate
|
| 141 |
+
reference blocks by segment table rather than by recomputing cursor
|
| 142 |
+
arithmetic, which made multiple reference blocks work cleanly. Ref2VA
|
| 143 |
+
graphs can carry their own image, video and audio references alongside
|
| 144 |
+
a chained head.
|
| 145 |
+
- **Latent picture path.** The pinned run is sliced straight out of the
|
| 146 |
+
previous clip's latent, skipping the decode and re-encode round trip
|
| 147 |
+
that dulls the picture a little more at every link.
|
| 148 |
+
- **Patches install on first use, not at import.** Having the pack in
|
| 149 |
+
`custom_nodes` now changes nothing until a Motion Context node
|
| 150 |
+
actually runs.
|
| 151 |
+
- **Coexistence detection.** A second copy of the patch, vendored into
|
| 152 |
+
another pack, recognises the first and stands down instead of
|
| 153 |
+
wrapping it.
|
| 154 |
+
- The payload patch is gated on this pack's own markers, so unrelated H3
|
| 155 |
+
graphs stay bit-identical to stock.
|
| 156 |
+
- Two settings exposed; everything with only one correct answer was
|
| 157 |
+
removed from the UI.
|
| 158 |
+
- `freeze_detect.py` and `level_step.py` added to the measurement
|
| 159 |
+
scripts.
|
| 160 |
+
- The example workflow was replaced with an fl2va / ref2va one.
|
| 161 |
+
|
| 162 |
+
## Initial release - 2026-08-06
|
| 163 |
+
|
| 164 |
+
Works with the older H3 layout only. Not published to the registry; no
|
| 165 |
+
version number, as `pyproject.toml` came later.
|
| 166 |
+
|
| 167 |
+
Clip chaining for MiniMax H3 with audio continuation rather than
|
| 168 |
+
imitation. Four nodes: Motion Context, Trim, Save Latent, Load Latent.
|
| 169 |
+
The pinned audio window is placed so it ends at the join and reaches
|
| 170 |
+
backwards, which is what makes the model continue a soundtrack instead
|
| 171 |
+
of starting something that merely resembles it.
|
custom_nodes/ComfyUI-H3-Motion-Context-main/LICENSE
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Copyright (C) 2026 NikoDemon80
|
| 2 |
+
|
| 3 |
+
GNU GENERAL PUBLIC LICENSE
|
| 4 |
+
Version 3, 29 June 2007
|
| 5 |
+
|
| 6 |
+
Copyright © 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
| 7 |
+
|
| 8 |
+
Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.
|
| 9 |
+
|
| 10 |
+
Preamble
|
| 11 |
+
|
| 12 |
+
The GNU General Public License is a free, copyleft license for software and other kinds of works.
|
| 13 |
+
|
| 14 |
+
The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too.
|
| 15 |
+
|
| 16 |
+
When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things.
|
| 17 |
+
|
| 18 |
+
To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others.
|
| 19 |
+
|
| 20 |
+
For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights.
|
| 21 |
+
|
| 22 |
+
Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it.
|
| 23 |
+
|
| 24 |
+
For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions.
|
| 25 |
+
|
| 26 |
+
Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users.
|
| 27 |
+
|
| 28 |
+
Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free.
|
| 29 |
+
|
| 30 |
+
The precise terms and conditions for copying, distribution and modification follow.
|
| 31 |
+
|
| 32 |
+
TERMS AND CONDITIONS
|
| 33 |
+
|
| 34 |
+
0. Definitions.
|
| 35 |
+
|
| 36 |
+
“This License” refers to version 3 of the GNU General Public License.
|
| 37 |
+
|
| 38 |
+
“Copyright” also means copyright-like laws that apply to other kinds of works, such as semiconductor masks.
|
| 39 |
+
|
| 40 |
+
“The Program” refers to any copyrightable work licensed under this License. Each licensee is addressed as “you”. “Licensees” and “recipients” may be individuals or organizations.
|
| 41 |
+
|
| 42 |
+
To “modify” a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a “modified version” of the earlier work or a work “based on” the earlier work.
|
| 43 |
+
|
| 44 |
+
A “covered work” means either the unmodified Program or a work based on the Program.
|
| 45 |
+
|
| 46 |
+
To “propagate” a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well.
|
| 47 |
+
|
| 48 |
+
To “convey” a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying.
|
| 49 |
+
|
| 50 |
+
An interactive user interface displays “Appropriate Legal Notices” to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion.
|
| 51 |
+
|
| 52 |
+
1. Source Code.
|
| 53 |
+
The “source code” for a work means the preferred form of the work for making modifications to it. “Object code” means any non-source form of a work.
|
| 54 |
+
|
| 55 |
+
A “Standard Interface” means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language.
|
| 56 |
+
|
| 57 |
+
The “System Libraries” of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A “Major Component”, in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it.
|
| 58 |
+
|
| 59 |
+
The “Corresponding Source” for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work.
|
| 60 |
+
|
| 61 |
+
The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source.
|
| 62 |
+
|
| 63 |
+
The Corresponding Source for a work in source code form is that same work.
|
| 64 |
+
|
| 65 |
+
2. Basic Permissions.
|
| 66 |
+
All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law.
|
| 67 |
+
|
| 68 |
+
You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you.
|
| 69 |
+
|
| 70 |
+
Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary.
|
| 71 |
+
|
| 72 |
+
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
| 73 |
+
No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures.
|
| 74 |
+
|
| 75 |
+
When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures.
|
| 76 |
+
|
| 77 |
+
4. Conveying Verbatim Copies.
|
| 78 |
+
You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program.
|
| 79 |
+
|
| 80 |
+
You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee.
|
| 81 |
+
|
| 82 |
+
5. Conveying Modified Source Versions.
|
| 83 |
+
You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions:
|
| 84 |
+
|
| 85 |
+
a) The work must carry prominent notices stating that you modified it, and giving a relevant date.
|
| 86 |
+
|
| 87 |
+
b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to “keep intact all notices”.
|
| 88 |
+
|
| 89 |
+
c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it.
|
| 90 |
+
|
| 91 |
+
d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so.
|
| 92 |
+
|
| 93 |
+
A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an “aggregate” if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate.
|
| 94 |
+
|
| 95 |
+
6. Conveying Non-Source Forms.
|
| 96 |
+
You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways:
|
| 97 |
+
|
| 98 |
+
a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange.
|
| 99 |
+
|
| 100 |
+
b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge.
|
| 101 |
+
|
| 102 |
+
c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b.
|
| 103 |
+
|
| 104 |
+
d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements.
|
| 105 |
+
|
| 106 |
+
e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d.
|
| 107 |
+
|
| 108 |
+
A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work.
|
| 109 |
+
|
| 110 |
+
A “User Product” is either (1) a “consumer product”, which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, “normally used” refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product.
|
| 111 |
+
|
| 112 |
+
“Installation Information” for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made.
|
| 113 |
+
|
| 114 |
+
If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM).
|
| 115 |
+
|
| 116 |
+
The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network.
|
| 117 |
+
|
| 118 |
+
Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying.
|
| 119 |
+
|
| 120 |
+
7. Additional Terms.
|
| 121 |
+
“Additional permissions” are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions.
|
| 122 |
+
|
| 123 |
+
When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission.
|
| 124 |
+
|
| 125 |
+
Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms:
|
| 126 |
+
|
| 127 |
+
a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or
|
| 128 |
+
|
| 129 |
+
b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or
|
| 130 |
+
|
| 131 |
+
c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or
|
| 132 |
+
|
| 133 |
+
d) Limiting the use for publicity purposes of names of licensors or authors of the material; or
|
| 134 |
+
|
| 135 |
+
e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or
|
| 136 |
+
|
| 137 |
+
f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors.
|
| 138 |
+
|
| 139 |
+
All other non-permissive additional terms are considered “further restrictions” within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying.
|
| 140 |
+
|
| 141 |
+
If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms.
|
| 142 |
+
|
| 143 |
+
Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way.
|
| 144 |
+
|
| 145 |
+
8. Termination.
|
| 146 |
+
You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11).
|
| 147 |
+
|
| 148 |
+
However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation.
|
| 149 |
+
|
| 150 |
+
Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice.
|
| 151 |
+
|
| 152 |
+
Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10.
|
| 153 |
+
|
| 154 |
+
9. Acceptance Not Required for Having Copies.
|
| 155 |
+
You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so.
|
| 156 |
+
|
| 157 |
+
10. Automatic Licensing of Downstream Recipients.
|
| 158 |
+
Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License.
|
| 159 |
+
|
| 160 |
+
An “entity transaction” is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts.
|
| 161 |
+
|
| 162 |
+
You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it.
|
| 163 |
+
|
| 164 |
+
11. Patents.
|
| 165 |
+
A “contributor” is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's “contributor version”.
|
| 166 |
+
|
| 167 |
+
A contributor's “essential patent claims” are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, “control” includes the right to grant patent sublicenses in a manner consistent with the requirements of this License.
|
| 168 |
+
|
| 169 |
+
Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version.
|
| 170 |
+
|
| 171 |
+
In the following three paragraphs, a “patent license” is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To “grant” such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party.
|
| 172 |
+
|
| 173 |
+
If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. “Knowingly relying” means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid.
|
| 174 |
+
|
| 175 |
+
If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it.
|
| 176 |
+
|
| 177 |
+
A patent license is “discriminatory” if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007.
|
| 178 |
+
|
| 179 |
+
Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law.
|
| 180 |
+
|
| 181 |
+
12. No Surrender of Others' Freedom.
|
| 182 |
+
If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program.
|
| 183 |
+
|
| 184 |
+
13. Use with the GNU Affero General Public License.
|
| 185 |
+
Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such.
|
| 186 |
+
|
| 187 |
+
14. Revised Versions of this License.
|
| 188 |
+
The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns.
|
| 189 |
+
|
| 190 |
+
Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License “or any later version” applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation.
|
| 191 |
+
|
| 192 |
+
If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program.
|
| 193 |
+
|
| 194 |
+
Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version.
|
| 195 |
+
|
| 196 |
+
15. Disclaimer of Warranty.
|
| 197 |
+
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
| 198 |
+
|
| 199 |
+
16. Limitation of Liability.
|
| 200 |
+
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
|
| 201 |
+
|
| 202 |
+
17. Interpretation of Sections 15 and 16.
|
| 203 |
+
If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee.
|
| 204 |
+
|
| 205 |
+
END OF TERMS AND CONDITIONS
|
| 206 |
+
|
| 207 |
+
How to Apply These Terms to Your New Programs
|
| 208 |
+
|
| 209 |
+
If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms.
|
| 210 |
+
|
| 211 |
+
To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the “copyright” line and a pointer to where the full notice is found.
|
| 212 |
+
|
| 213 |
+
<one line to give the program's name and a brief idea of what it does.>
|
| 214 |
+
Copyright (C) <year> <name of author>
|
| 215 |
+
|
| 216 |
+
This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
|
| 217 |
+
|
| 218 |
+
This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
| 219 |
+
|
| 220 |
+
You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>.
|
| 221 |
+
|
| 222 |
+
Also add information on how to contact you by electronic and paper mail.
|
| 223 |
+
|
| 224 |
+
If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode:
|
| 225 |
+
|
| 226 |
+
<program> Copyright (C) <year> <name of author>
|
| 227 |
+
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
|
| 228 |
+
This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details.
|
| 229 |
+
|
| 230 |
+
The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an “about box”.
|
| 231 |
+
|
| 232 |
+
You should also get your employer (if you work as a programmer) or school, if any, to sign a “copyright disclaimer” for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see <https://www.gnu.org/licenses/>.
|
| 233 |
+
|
| 234 |
+
The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read <https://www.gnu.org/philosophy/why-not-lgpl.html>.
|
custom_nodes/ComfyUI-H3-Motion-Context-main/README.md
ADDED
|
@@ -0,0 +1,477 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# H3 Motion Context
|
| 2 |
+
|
| 3 |
+
Chain MiniMax H3 clips so motion and sound keep going across the cut.
|
| 4 |
+
|
| 5 |
+
Generate clip A. Feed its last frames and audio into this node. Generate
|
| 6 |
+
clip B. B picks up where A left off: same motion, same speed, same
|
| 7 |
+
direction, and the same audio continued rather than a new take that
|
| 8 |
+
sounds similar. Repeat as long as you like.
|
| 9 |
+
|
| 10 |
+
Nothing on disk is edited and nothing in ComfyUI is modified. The nodes
|
| 11 |
+
check their arithmetic against the live ComfyUI code before the first
|
| 12 |
+
render, and if an update breaks an assumption they refuse to run and say
|
| 13 |
+
what moved. A loud failure beats a bad render you don't notice.
|
| 14 |
+
|
| 15 |
+
Requires ComfyUI 0.34.0 or newer, which is where H3 gained arbitrary
|
| 16 |
+
keyframe anchors. Version 0.3.1 runs on that and on everything older;
|
| 17 |
+
see [CHANGELOG.md](CHANGELOG.md).
|
| 18 |
+
|
| 19 |
+
## Why this exists
|
| 20 |
+
|
| 21 |
+
ComfyUI can anchor a still or a short clip at any frame of an H3
|
| 22 |
+
generation, with the Add Guide for MiniMax H3 node. If that is all you
|
| 23 |
+
need, use it. You do not need this pack for it.
|
| 24 |
+
|
| 25 |
+
Chaining is a different problem, and two parts of it are still unsolved
|
| 26 |
+
by anything else.
|
| 27 |
+
|
| 28 |
+
**The picture goes out to pixels and back at every link.** Add Guide takes
|
| 29 |
+
images and encodes them. Chain through it and every join costs a decode, a
|
| 30 |
+
resize and a re-encode. That round trip is where the colour drift and the
|
| 31 |
+
softening down a long chain come from. This pack slices the previous
|
| 32 |
+
clip's tail straight out of its latent, so the pinned frames are the same
|
| 33 |
+
numbers they were, bit for bit.
|
| 34 |
+
|
| 35 |
+
**The sound restarts instead of continuing.** Add Guide anchors audio
|
| 36 |
+
starting at a frame and running forward. To actually continue a
|
| 37 |
+
soundtrack, the pinned window has to END at the join and reach backwards
|
| 38 |
+
into the sound that already played. That is the difference between the
|
| 39 |
+
model continuing your track and the model writing something that sounds
|
| 40 |
+
like it, and on anything with a beat you hear it immediately.
|
| 41 |
+
|
| 42 |
+
Everything else here follows from those two: the Trim node that removes
|
| 43 |
+
the pinned head from the delivery, the Save/Load Latent pair that carries
|
| 44 |
+
a clip across without touching pixels, and the Seam Probe that measures
|
| 45 |
+
whether a join is a real continuation or a convincing imitation.
|
| 46 |
+
|
| 47 |
+
Audio was the harder half, and it's the more useful half, since H3
|
| 48 |
+
generates picture and sound together. See "Why the audio needed work"
|
| 49 |
+
below if you care how.
|
| 50 |
+
|
| 51 |
+
## Install
|
| 52 |
+
|
| 53 |
+
Drop the folder in `ComfyUI/custom_nodes/` and restart. At startup you'll
|
| 54 |
+
just see:
|
| 55 |
+
|
| 56 |
+
```
|
| 57 |
+
h3_motion_context: nodes registered. ComfyUI is not modified; the layout
|
| 58 |
+
checks run on the first use of a Motion Context node.
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
Having the pack installed changes nothing about your other H3 workflows,
|
| 62 |
+
and nothing runs until you actually chain a clip. The first time you do,
|
| 63 |
+
you'll see:
|
| 64 |
+
|
| 65 |
+
```
|
| 66 |
+
h3_motion_context: ComfyUI H3 layout checks passed, anchors and pinned audio will land where intended
|
| 67 |
+
```
|
| 68 |
+
|
| 69 |
+
Anything else and the node refuses to run. The reason is logged.
|
| 70 |
+
|
| 71 |
+
On ComfyUI 0.33.4 or older the node says so and points you at version
|
| 72 |
+
0.3.1, which runs on both. That is read from the layout code itself
|
| 73 |
+
rather than a version number, so a nightly or a fork gets the right
|
| 74 |
+
answer.
|
| 75 |
+
|
| 76 |
+
## Wiring
|
| 77 |
+
|
| 78 |
+
```
|
| 79 |
+
MiniMaxH3ImageToVideo / MiniMaxH3ReferenceToVideo (or the t2v path)
|
| 80 |
+
-> H3 Motion Context <- previous clip's latent (picture + sound)
|
| 81 |
+
-> guider / sampler
|
| 82 |
+
...
|
| 83 |
+
decoded IMAGE + AUDIO
|
| 84 |
+
-> H3 Motion Context Trim <- wire trim_frames across
|
| 85 |
+
-> Create Video / save
|
| 86 |
+
```
|
| 87 |
+
|
| 88 |
+
Wire `trim_frames` into the Trim node. The pinned frames come back at the
|
| 89 |
+
start of the new clip and have to come off before you concatenate, picture
|
| 90 |
+
and sound together.
|
| 91 |
+
|
| 92 |
+
### Carrying the previous clip across
|
| 93 |
+
|
| 94 |
+
The previous clip reaches the node as a latent, but you can't wire the sampler
|
| 95 |
+
straight into `context_latent`. ComfyUI will call it a circular
|
| 96 |
+
connection, and it's right: the latent you want is from the previous run,
|
| 97 |
+
not this one. Two helper nodes move it across runs the same way your
|
| 98 |
+
frames and audio already move across, through a file:
|
| 99 |
+
|
| 100 |
+
```
|
| 101 |
+
this run: SamplerCustomAdvanced -> H3 Motion Context Save Latent
|
| 102 |
+
next run: H3 Motion Context Load Latent -> context_latent
|
| 103 |
+
```
|
| 104 |
+
|
| 105 |
+
Both have a `clip_index` and the numbers mean what they say. On Load, the
|
| 106 |
+
clip you're continuing FROM. On Save, the clip this one IS. First clip:
|
| 107 |
+
Load 0, Save 1. Making clip 2 from clip 1: Load 1, Save 2.
|
| 108 |
+
|
| 109 |
+
Load 0 does not read a file. Motion Context then passes the incoming
|
| 110 |
+
conditioning through and reports `trim_frames` 0, so you can leave the
|
| 111 |
+
node enabled for the first clip. Do not mute it.
|
| 112 |
+
|
| 113 |
+
Do not use queue **run on change** to walk the chain. Each index change is
|
| 114 |
+
a separate graph edit, so two widgets incrementing will queue two runs and
|
| 115 |
+
skip slots. Use **H3 Motion Context Chain** instead of ComfyUI's Run
|
| 116 |
+
button.
|
| 117 |
+
|
| 118 |
+
**The Chain node only sees Load and Save if all three sit in the same
|
| 119 |
+
canvas group** (one of those coloured boxes you draw around nodes). If
|
| 120 |
+
they are not grouped together, every button does nothing. Select Load,
|
| 121 |
+
Save, and Chain, then Group Selected (or drop Chain into the existing
|
| 122 |
+
Motion Context group).
|
| 123 |
+
|
| 124 |
+
- **Run/Re-roll** is the Run button for this graph. It queues at the
|
| 125 |
+
current Load/Save indices and does not advance them. First clip: leave
|
| 126 |
+
Load 0 / Save 1 and click it. Hate the result? Click it again; the
|
| 127 |
+
retry overwrites that slot.
|
| 128 |
+
- **Approve** advances both indices, then queues the next clip once.
|
| 129 |
+
- **Chain** is Approve on a loop. After a finished clip it bumps
|
| 130 |
+
Load/Save and keeps going, so a 0/1 clip you already made becomes 1/2
|
| 131 |
+
rather than a re-roll. The exception is Load 0 / Save 1 with no clip 1
|
| 132 |
+
on disk: it generates that first clip, then auto-approves. **segments**
|
| 133 |
+
is how many clips that loop runs: 5 means five clips then stop, 0 means
|
| 134 |
+
keep going until you click **Stop**.
|
| 135 |
+
- **Reset** sets Load 0 / Save 1. It does not queue and does not delete
|
| 136 |
+
files.
|
| 137 |
+
- **Clear latents** deletes numbered chain slots
|
| 138 |
+
(`clip_00001.safetensors` and so on) in the Load folder. Files you
|
| 139 |
+
renamed are left alone. Indices stay put.
|
| 140 |
+
|
| 141 |
+
Files are named the obvious way, `clip_00002.safetensors` is clip 2.
|
| 142 |
+
|
| 143 |
+
Auto-saved files (Save at 0) get a trailing underscore
|
| 144 |
+
(`clip_00002_.safetensors`) because they're numbered by run, not by clip,
|
| 145 |
+
and indexed loading skips them on purpose.
|
| 146 |
+
|
| 147 |
+
You can also point the loader at a specific file when `clip_index` is
|
| 148 |
+
greater than 0; the file is loaded and the index is ignored. Index 0
|
| 149 |
+
never reads a file.
|
| 150 |
+
Its output is only for `context_latent`. Don't wire it into a decode node.
|
| 151 |
+
Stock Save/Load Latent won't work here, it can't handle H3's paired
|
| 152 |
+
video/audio latent.
|
| 153 |
+
|
| 154 |
+
The latent carries both streams, so with it wired you don't need to load
|
| 155 |
+
the previous clip's video at all. The pinned frames are sliced straight
|
| 156 |
+
out of it rather than decoded to pixels and encoded again, so they're
|
| 157 |
+
exactly what the model made instead of a reconstruction. Nothing shifts
|
| 158 |
+
the colour or the contrast, so there's no seam to see, and it's faster.
|
| 159 |
+
|
| 160 |
+
Nothing to set for any of that. Leave `context_frames` unwired. Resolution
|
| 161 |
+
has to match between clips, since a latent can't be resized; if it doesn't
|
| 162 |
+
the node refuses and names both resolutions rather than quietly dropping
|
| 163 |
+
to the lossy path.
|
| 164 |
+
|
| 165 |
+
There's an older path for graphs with no latent: the previous clip's
|
| 166 |
+
frames into `context_frames`, its decoded audio into `context_audio`, and
|
| 167 |
+
the H3 audio VAE into `audio_vae`. It works, but it costs a lossy round
|
| 168 |
+
trip per link on both streams and it's where the visible seams came from.
|
| 169 |
+
Use the latent unless you have a reason not to.
|
| 170 |
+
|
| 171 |
+
### Reference mode
|
| 172 |
+
|
| 173 |
+
Put this node after `MiniMaxH3ReferenceToVideo` and wire it as above. Your
|
| 174 |
+
reference blocks (image, video, video with audio, audio) are left alone
|
| 175 |
+
entirely: the pinned sound rides as a keyframe, not a reference, so the
|
| 176 |
+
two mechanisms no longer share a list.
|
| 177 |
+
|
| 178 |
+
Nothing to configure. Worth mentioning only because versions before 0.2.0
|
| 179 |
+
overwrote that list, so turning chaining on quietly threw your references
|
| 180 |
+
away, and because up to 0.3.1 the pinned audio was added to it.
|
| 181 |
+
|
| 182 |
+
References still push the target timeline along, and the pinned frames
|
| 183 |
+
follow it. That is ComfyUI's own arithmetic now rather than this pack's.
|
| 184 |
+
|
| 185 |
+
### Alongside Add Guide
|
| 186 |
+
|
| 187 |
+
Anchor whatever you like with Add Guide for MiniMax H3 and put this node
|
| 188 |
+
after it. Both survive: the pinned head decides how the clip starts, the
|
| 189 |
+
guide decides what happens where you put it. Guides that carry audio keep
|
| 190 |
+
their own placement, which is forward from the anchor frame, while the
|
| 191 |
+
pinned sound keeps its backwards-reaching window.
|
| 192 |
+
|
| 193 |
+
Guides anchored inside the pinned head are dropped with a warning, for
|
| 194 |
+
the same reason a `first_frame` anchor is.
|
| 195 |
+
|
| 196 |
+
### Keeping a last-frame target
|
| 197 |
+
|
| 198 |
+
Wire `last_frame` on the stock conditioning node as usual and put this
|
| 199 |
+
node after it. The anchor is kept and pinned alongside the head: the
|
| 200 |
+
pinned run decides how the clip starts, your image decides where it
|
| 201 |
+
ends. It keeps the coordinates ComfyUI gave it, including the shift
|
| 202 |
+
references apply, so it stays put whatever else is in the graph.
|
| 203 |
+
|
| 204 |
+
A `first_frame` anchor is dropped, with a warning. The pinned head owns
|
| 205 |
+
those frames, and a second block pinned at the same instant with
|
| 206 |
+
different content would fight it. Older versions silently replaced both
|
| 207 |
+
anchors, so if a chain graph carried a last-frame target before this
|
| 208 |
+
version, it never reached the model.
|
| 209 |
+
|
| 210 |
+
## Settings
|
| 211 |
+
|
| 212 |
+
Two, because everything else had exactly one right answer.
|
| 213 |
+
|
| 214 |
+
**context_length** - frames of the previous clip's picture to carry over.
|
| 215 |
+
5, 22, 39 or 56. Those are the lengths that are a whole number of latent
|
| 216 |
+
steps, which is why the others aren't offered. 5 is just barely fluid, 22
|
| 217 |
+
is nearly seamless. Longer windows pin more motion but they come off the
|
| 218 |
+
front of the delivered clip, so 56 spends 2.3 seconds of every render on
|
| 219 |
+
frames you throw away. **Use 22.**
|
| 220 |
+
|
| 221 |
+
**audio_context_length** - frames of tail audio to pin, independent of the
|
| 222 |
+
picture window. It ends at the same instant as the pinned video, so this
|
| 223 |
+
only controls how far back the sound reaches. 0 follows context_length.
|
| 224 |
+
**Use 24**: that is exactly one second of sound, and any multiple of 3
|
| 225 |
+
lands exactly on the model's 40 Hz audio grid (a frame is 5/3 of an
|
| 226 |
+
audio step), so the window is pinned at precisely the width you asked
|
| 227 |
+
for. Off-grid values are widened to the nearest whole step. Increments
|
| 228 |
+
of 24 keep whole seconds: 48 pins the last two.
|
| 229 |
+
|
| 230 |
+
Everything else is fixed: the pinned run is encoded in one VAE call, it
|
| 231 |
+
sits at the head of the clip where the Trim node removes it, and the
|
| 232 |
+
pinned audio goes on this clip's own timeline. The alternatives all
|
| 233 |
+
existed only to reproduce their own failures, so they're constants at the
|
| 234 |
+
top of `nodes.py` now. Change one there if you ever need to see what they
|
| 235 |
+
did.
|
| 236 |
+
|
| 237 |
+
`match_tail` on the Trim node stays a setting because that node has no
|
| 238 |
+
idea what the other one did. Leave it on. H3 rounds its audio grid up, so
|
| 239 |
+
every clip carries about 8ms more sound than picture, and that error
|
| 240 |
+
stacks at every join.
|
| 241 |
+
|
| 242 |
+
## Writing prompts for a chain
|
| 243 |
+
|
| 244 |
+
The settings get motion and sound across the join. What happens in the
|
| 245 |
+
next clip is on you, and there are a few traps.
|
| 246 |
+
|
| 247 |
+
**The model renders contradictions as unions.** Clip N ends on a close-up
|
| 248 |
+
of A, clip N+1's prompt opens with "a two-shot of B and C," and the model
|
| 249 |
+
doesn't pick. You get all three. The pinned frames aren't a suggestion, so
|
| 250 |
+
a prompt describing a different arrangement of people reads as an addition
|
| 251 |
+
to them, not a replacement.
|
| 252 |
+
|
| 253 |
+
**The airlock.** Don't ask for the change and the continuation at the same
|
| 254 |
+
moment. Open clip N+1 holding clip N's exact closing framing, no dialogue,
|
| 255 |
+
about two seconds, then cut to the new setup. Joins done this way measure
|
| 256 |
+
tighter than an ordinary frame-to-frame cut. Joins that skip it measure
|
| 257 |
+
like two different rooms spliced together.
|
| 258 |
+
|
| 259 |
+
**Give the hold something to do.** A held framing with nothing happening
|
| 260 |
+
renders as a literal freeze, and two seconds of a motionless actor looks
|
| 261 |
+
like the video stalled. Write in a breath, a weight shift, an eyeline
|
| 262 |
+
change. The camera holds still, the performer doesn't.
|
| 263 |
+
`tests/freeze_detect.py` finds these.
|
| 264 |
+
|
| 265 |
+
**Budget the pinned head in your timecodes.** In `head` mode the clip
|
| 266 |
+
comes out `context_length` frames shorter than it was sampled. At 22 that's
|
| 267 |
+
0.92 seconds. Your prompt timings land against the sampled version, which
|
| 268 |
+
starts 0.92s earlier, so a beat you wrote for 4.0s shows up at 3.08s in the
|
| 269 |
+
file.
|
| 270 |
+
|
| 271 |
+
## Why the audio needed work
|
| 272 |
+
|
| 273 |
+
The first version ran pinned audio through H3's reference mechanism, which
|
| 274 |
+
is where audio conditioning normally goes. Every join had a small tick,
|
| 275 |
+
like the audio briefly sped up and went offbeat. Looking at the waveform
|
| 276 |
+
showed nothing wrong. Both sides of every join were smooth on their own.
|
| 277 |
+
|
| 278 |
+
Cross-correlating each clip's opening against the previous clip's ending
|
| 279 |
+
(that's `tests/seam_probe.py`) showed what was actually happening: the new
|
| 280 |
+
clip's audio resembled the old one. Same instruments, same groove, never
|
| 281 |
+
the same recording. A cover band. The model was reading the reference as
|
| 282 |
+
"a separate clip that sounds like this," which is what references are for
|
| 283 |
+
and exactly wrong for continuation.
|
| 284 |
+
|
| 285 |
+
The fix is the same one that already worked for video. The rows the model
|
| 286 |
+
sees are identical either way. What differs is their time coordinates, and
|
| 287 |
+
the coordinates are what say "separate clip" versus "this clip, earlier."
|
| 288 |
+
So the pinned audio still rides the reference machinery, but its
|
| 289 |
+
coordinates get rewritten onto the new clip's timeline, ending exactly
|
| 290 |
+
where the pinned video ends. Correlation at the joins went from about 0.45
|
| 291 |
+
with incoherent timing to 0.95+ with a flat offset, and the tick was gone.
|
| 292 |
+
Measured across a chain, the offset doesn't grow from join to join.
|
| 293 |
+
|
| 294 |
+
## Limitations
|
| 295 |
+
|
| 296 |
+
**Quality degrades down a chain.** The big one, and it's mostly audio.
|
| 297 |
+
Each clip is generated from the previous clip's output, which came from
|
| 298 |
+
the one before it. Losses compound like photocopying a photocopy, and in
|
| 299 |
+
audio the top end goes first. Timing and tempo stay locked, but after
|
| 300 |
+
several clips the sound gets duller and more muffled. Picture holds up
|
| 301 |
+
much better.
|
| 302 |
+
|
| 303 |
+
Two things stack per link: the model's own smoothing, and a round trip
|
| 304 |
+
out to pixels and back. `context_latent` removes the second one for both
|
| 305 |
+
streams. On picture that's the difference between a visible join and no
|
| 306 |
+
join at all. On sound it helps, but the model's own smoothing is still
|
| 307 |
+
there. Long chains are worth listening to critically, and restarts land
|
| 308 |
+
best at a natural musical transition.
|
| 309 |
+
|
| 310 |
+
**A small audio offset, now believed fixed.** Chained clips used to come
|
| 311 |
+
out about 8ms late. The cause was arithmetic: a frame is 5/3 of an audio
|
| 312 |
+
step, so unless the pinned window's length works out to a whole number of
|
| 313 |
+
steps it landed between the ones the model was filling. The window is now
|
| 314 |
+
snapped to that grid. The offset was constant, well under where lip-sync
|
| 315 |
+
errors get noticeable, and it didn't grow down a chain, so this is a
|
| 316 |
+
tidy-up rather than a rescue. `tests/seam_probe.py` reports the lag in
|
| 317 |
+
milliseconds if you want to check it on your own material.
|
| 318 |
+
|
| 319 |
+
**H3 emits 32 kHz audio, not 48.** Read the rate off the clip in any script
|
| 320 |
+
that remuxes or concatenates. A stream-copy concat can't change rate
|
| 321 |
+
partway, so a hardcoded 48000 silently turns the tail of a long episode
|
| 322 |
+
into nothing while every duration check still passes.
|
| 323 |
+
`tests/level_step.py` prints every clip's rate and flags a mismatch.
|
| 324 |
+
|
| 325 |
+
**Turbo LoRAs and Spectrum both cost you audio.** A turbo LoRA hits a
|
| 326 |
+
result in very few steps, and fine detail is what those last steps were
|
| 327 |
+
for. It thickens the sound and softens the picture. Step-skipping
|
| 328 |
+
optimizers like ComfyUI-Spectrum-MiniMax-H3 do the same thing to the
|
| 329 |
+
audio, less so to the picture, and they also mispredict the pinned rows,
|
| 330 |
+
which never change. Run both together and it stacks. If a chain sounds
|
| 331 |
+
duller or closer than you expected, try turning these off before blaming
|
| 332 |
+
the chaining. **Keep Spectrum off for these graphs.**
|
| 333 |
+
|
| 334 |
+
**Resolution can't change mid-chain** while using `context_latent`. A
|
| 335 |
+
latent can't be resized, so the node refuses. Regenerate the previous clip
|
| 336 |
+
at the new resolution, or start a fresh chain there.
|
| 337 |
+
|
| 338 |
+
**Prompting a chain takes work**, especially with reference mode. Every
|
| 339 |
+
reference conditions the whole clip; there's no way to say "this one
|
| 340 |
+
starts two seconds in." Timing has to come from the shot structure and
|
| 341 |
+
the description. See "Writing prompts for a chain" above.
|
| 342 |
+
|
| 343 |
+
**Tested narrowly.** Joins have been verified on dense beat-driven
|
| 344 |
+
electronic music, where timing errors are most audible, and on spoken word
|
| 345 |
+
through the latent path, where nothing hides a seam. One Windows machine,
|
| 346 |
+
one resolution, one sampler. The math self-tests every startup. The
|
| 347 |
+
perceptual results are one person's renders.
|
| 348 |
+
|
| 349 |
+
**ComfyUI's H3 support is young.** This pack no longer patches any of it,
|
| 350 |
+
but it still depends on how the layout places things, and one of those
|
| 351 |
+
dependencies has no upstream test behind it: the pinned audio window is
|
| 352 |
+
positioned with a fractional, negative anchor index. That is legal
|
| 353 |
+
arithmetic in the layout and no stock node can produce one, so nothing
|
| 354 |
+
upstream exercises it. `layout_contract.py` checks it before the first
|
| 355 |
+
render and refuses if it ever stops holding.
|
| 356 |
+
|
| 357 |
+
The failure mode after a ComfyUI update is therefore "the node won't
|
| 358 |
+
run," not bad output. There is a worked example: when the layout
|
| 359 |
+
constructor changed, 0.3.0 of this pack refused until 0.3.1 caught up.
|
| 360 |
+
Annoying, but the alternative was joins landing at the wrong instant with
|
| 361 |
+
nothing to tell you.
|
| 362 |
+
|
| 363 |
+
**License.** The H3 community license reportedly doesn't currently cover
|
| 364 |
+
the EU, UK, Korea or the US. Check for yourself before shipping anything
|
| 365 |
+
on it.
|
| 366 |
+
|
| 367 |
+
## Recommended starting point
|
| 368 |
+
|
| 369 |
+
`context_length 22`, `audio_context_length 24`, `context_latent` wired
|
| 370 |
+
through the Save/Load Latent pair, Trim node wired for picture and sound
|
| 371 |
+
with `match_tail` on, Spectrum off. Every "it works" in this README means
|
| 372 |
+
that config.
|
| 373 |
+
|
| 374 |
+
## Testing
|
| 375 |
+
|
| 376 |
+
Six scripts, all runnable without ComfyUI or a GPU.
|
| 377 |
+
|
| 378 |
+
```
|
| 379 |
+
python tests/_mock_harness.py # the layout checks against a fake stock model
|
| 380 |
+
python tests/_node_smoke_test.py # the node end to end, refs + save/load
|
| 381 |
+
python tests/_probe_node_test.py # the seam probe node, joins with known answers
|
| 382 |
+
python tests/seam_probe.py A.flac B_untrimmed.flac # is the join real continuation?
|
| 383 |
+
python tests/level_step.py clip*.flac # does the level or room tone jump?
|
| 384 |
+
python tests/freeze_detect.py clip*.mp4 # did a held shot render as a still?
|
| 385 |
+
```
|
| 386 |
+
|
| 387 |
+
The first three print their checks and end with a pass line. The other
|
| 388 |
+
three measure real output; `level_step` and `freeze_detect` take
|
| 389 |
+
`--self-test` to check their own math on made-up data first, and
|
| 390 |
+
`seam_probe`'s math is what `_probe_node_test.py` exercises.
|
| 391 |
+
|
| 392 |
+
The three measurement scripts ask different questions and a join can fail
|
| 393 |
+
any one on its own. `seam_probe` is timing: is the audio the same waveform
|
| 394 |
+
continued, or a sound-alike. `level_step` is volume: does the loudness, or
|
| 395 |
+
the room tone underneath it, jump at the cut. `freeze_detect` is picture:
|
| 396 |
+
is anything happening in that shot.
|
| 397 |
+
|
| 398 |
+
For `seam_probe` and `level_step`, give them the UNTRIMMED audio of the new
|
| 399 |
+
clip. Branch the audio decode to a second Save Audio node alongside the
|
| 400 |
+
Trim.
|
| 401 |
+
|
| 402 |
+
### Measuring a join in-graph
|
| 403 |
+
|
| 404 |
+
The H3 Motion Context Seam Probe node runs the same measurements inside
|
| 405 |
+
the graph, where the seam position is known exactly rather than inferred
|
| 406 |
+
from file ends (that inference is where the CLI probe's phantom ~8 ms lag
|
| 407 |
+
came from). Wire it inline between the audio VAE decode and the Trim
|
| 408 |
+
node:
|
| 409 |
+
|
| 410 |
+
```
|
| 411 |
+
clip_a_latent the same latent wired into context_latent
|
| 412 |
+
audio_vae the same audio VAE
|
| 413 |
+
clip_b_untrimmed this clip's audio off the VAE decode, before the trim
|
| 414 |
+
trim_frames from the Motion Context node, same value the Trim gets
|
| 415 |
+
```
|
| 416 |
+
|
| 417 |
+
The audio output is the input unchanged, so it drops into an existing
|
| 418 |
+
chain without changing the render. The report goes to a Preview Text
|
| 419 |
+
node: lag, correlation, broadband and floor steps, judged against the
|
| 420 |
+
same thresholds the CLI scripts document.
|
| 421 |
+
|
| 422 |
+
## Upgrading
|
| 423 |
+
|
| 424 |
+
[CHANGELOG.md](CHANGELOG.md) lists what changed in each release and which
|
| 425 |
+
ComfyUI H3 layout it works with. Worth a look before opening an issue or
|
| 426 |
+
starting a fork: several fixes people have rebuilt from scratch were
|
| 427 |
+
already in an earlier release.
|
| 428 |
+
|
| 429 |
+
The node's widgets changed. ComfyUI stores widget values by position, so a
|
| 430 |
+
workflow saved against an older version will load its numbers into the
|
| 431 |
+
wrong slots. Delete the Motion Context node, add it again and rewire it.
|
| 432 |
+
Takes a minute and beats rendering with scrambled settings.
|
| 433 |
+
|
| 434 |
+
**Only install this once.** A fork of this repo, a manual clone next to a
|
| 435 |
+
Manager install, or a renamed backup still sitting in `custom_nodes` will
|
| 436 |
+
all load. Renaming a folder does not stop ComfyUI loading it.
|
| 437 |
+
|
| 438 |
+
Older versions of this pack, and other H3 packs, patch ComfyUI's layout
|
| 439 |
+
code at runtime. This one does not, so it no longer competes for that
|
| 440 |
+
code and will run alongside them. If it finds the layout wrapped it says
|
| 441 |
+
who by, then checks whether anchors still land correctly and carries on
|
| 442 |
+
if they do. Keeping one H3 chaining pack installed is still the quiet
|
| 443 |
+
life.
|
| 444 |
+
|
| 445 |
+
## Credits
|
| 446 |
+
|
| 447 |
+
The Ref2VA multi-reference support was worked out by **seitanism** in the
|
| 448 |
+
Banodoco MiniMax H3 seamless-extension thread and first implemented in
|
| 449 |
+
**ethanfel**'s fork of this repo. The code here is written independently,
|
| 450 |
+
but the idea is theirs and they got there first.
|
| 451 |
+
|
| 452 |
+
The prompting section comes out of a 16-clip, 4:34 multicam sitcom episode
|
| 453 |
+
chained with this pack at 736x576 on a 5070 Ti, about two hours of
|
| 454 |
+
generation. The airlock, the freeze warning, the 32 kHz trap and the
|
| 455 |
+
timecode budget are all that build's findings. Same run measured a
|
| 456 |
+
video-only chain at a median seam level step of 0.905, dropping to 0.16
|
| 457 |
+
with the audio carried across.
|
| 458 |
+
|
| 459 |
+
**javawock7618** and **azra1l** reported the layout change that broke
|
| 460 |
+
0.3.0 and narrowed it to specific commits, which turned a hunt into a
|
| 461 |
+
diff. **ChimeraWerks** found and fixed the audio grid overhang.
|
| 462 |
+
|
| 463 |
+
If you build something long with this, numbers are more useful than praise.
|
| 464 |
+
Open an issue.
|
| 465 |
+
|
| 466 |
+
## Files
|
| 467 |
+
|
| 468 |
+
| File | Role |
|
| 469 |
+
|---|---|
|
| 470 |
+
| `layout_contract.py` | Proves ComfyUI still places anchors and pinned audio where this pack needs them, once, before the first render. Modifies nothing. |
|
| 471 |
+
| `nodes.py` | The four core nodes: Motion Context, Trim, and the latent Save/Load pair. |
|
| 472 |
+
| `probe_node.py` | The Seam Probe node: measures a join in-graph, with the seam at a known sample instead of inferred from file ends. |
|
| 473 |
+
| `tests/seam_probe.py` | Is a join's audio a real continuation, a sound-alike, or drifting. |
|
| 474 |
+
| `tests/level_step.py` | Level and room-tone continuity at each join. Also catches sample-rate mismatches. |
|
| 475 |
+
| `tests/freeze_detect.py` | Stretches where the picture stops moving. |
|
| 476 |
+
| `tests/_mock_harness.py`, `tests/_node_smoke_test.py`, `tests/_probe_node_test.py` | Layout, node and probe tests, numpy only. |
|
| 477 |
+
| `CHANGELOG.md` | What changed in each release, and which ComfyUI H3 layout it works with. |
|
custom_nodes/ComfyUI-H3-Motion-Context-main/__init__.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""H3 Motion Context.
|
| 2 |
+
|
| 3 |
+
Clip chaining for MiniMax H3: pin the tail of the previous clip, picture
|
| 4 |
+
and sound, so the next clip genuinely continues it.
|
| 5 |
+
|
| 6 |
+
This registers the nodes and nothing else. Nothing in ComfyUI is
|
| 7 |
+
modified. Earlier versions wrapped ComfyUI's H3 layout constructor to
|
| 8 |
+
lift a restriction on where a keyframe could be anchored; ComfyUI 0.34.0
|
| 9 |
+
lifted it upstream, so the nodes now build plain keyframe dicts and hand
|
| 10 |
+
them to stock code.
|
| 11 |
+
|
| 12 |
+
What remains is a dependency on meaning rather than structure: the pinned
|
| 13 |
+
audio window is placed with a fractional, negative anchor index, which is
|
| 14 |
+
legal arithmetic in the layout but is not reachable through any stock
|
| 15 |
+
node and therefore has no upstream test. layout_contract.py checks it
|
| 16 |
+
before the first render and refuses if it ever stops holding, so an
|
| 17 |
+
upstream change produces a clear error rather than a silently wrong join.
|
| 18 |
+
|
| 19 |
+
That check runs on first use, not at import, so having the pack installed
|
| 20 |
+
changes nothing until you actually chain a clip.
|
| 21 |
+
|
| 22 |
+
Requires ComfyUI 0.34.0 or newer. Pack version 0.3.1 is the last release
|
| 23 |
+
that works with 0.33.4 and older.
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
import logging
|
| 27 |
+
|
| 28 |
+
from .nodes import (
|
| 29 |
+
NODE_CLASS_MAPPINGS,
|
| 30 |
+
NODE_DISPLAY_NAME_MAPPINGS,
|
| 31 |
+
register_chain_routes,
|
| 32 |
+
)
|
| 33 |
+
from .probe_node import (
|
| 34 |
+
NODE_CLASS_MAPPINGS as _PROBE_CLASSES,
|
| 35 |
+
NODE_DISPLAY_NAME_MAPPINGS as _PROBE_NAMES,
|
| 36 |
+
)
|
| 37 |
+
|
| 38 |
+
NODE_CLASS_MAPPINGS.update(_PROBE_CLASSES)
|
| 39 |
+
NODE_DISPLAY_NAME_MAPPINGS.update(_PROBE_NAMES)
|
| 40 |
+
register_chain_routes()
|
| 41 |
+
|
| 42 |
+
logging.getLogger("h3_motion_context").info(
|
| 43 |
+
"h3_motion_context: nodes registered. ComfyUI is not modified; the "
|
| 44 |
+
"layout checks run on the first use of a Motion Context node.")
|
| 45 |
+
|
| 46 |
+
WEB_DIRECTORY = "./web"
|
| 47 |
+
|
| 48 |
+
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"]
|
custom_nodes/ComfyUI-H3-Motion-Context-main/__pycache__/__init__.cpython-313.pyc
ADDED
|
Binary file (2.01 kB). View file
|
|
|
custom_nodes/ComfyUI-H3-Motion-Context-main/__pycache__/csrf_guard.cpython-313.pyc
ADDED
|
Binary file (4.12 kB). View file
|
|
|
custom_nodes/ComfyUI-H3-Motion-Context-main/__pycache__/layout_contract.cpython-313.pyc
ADDED
|
Binary file (12 kB). View file
|
|
|
custom_nodes/ComfyUI-H3-Motion-Context-main/__pycache__/nodes.cpython-313.pyc
ADDED
|
Binary file (48.8 kB). View file
|
|
|
custom_nodes/ComfyUI-H3-Motion-Context-main/__pycache__/probe_node.cpython-313.pyc
ADDED
|
Binary file (22.8 kB). View file
|
|
|
custom_nodes/ComfyUI-H3-Motion-Context-main/csrf_guard.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Same-origin gate for mutating Motion Context HTTP routes.
|
| 2 |
+
|
| 3 |
+
The Comfy Registry flags unauthenticated POST/DELETE that write or delete
|
| 4 |
+
files as CSRF-forgeable (policy-v0.2 UNAUTHENTICATED_SIDE_EFFECT). ComfyUI's
|
| 5 |
+
own loopback Origin/Host middleware is not part of this pack, so the
|
| 6 |
+
clear-latents handler must check the request itself.
|
| 7 |
+
|
| 8 |
+
A same-origin UI fetch sends Origin matching Host. A cross-site form POST
|
| 9 |
+
sends a different Origin, or Origin: null from a sandboxed iframe.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from functools import wraps
|
| 13 |
+
from urllib.parse import urlparse, urlsplit
|
| 14 |
+
|
| 15 |
+
from aiohttp import web
|
| 16 |
+
|
| 17 |
+
_MUTATING = frozenset({"POST", "PUT", "PATCH", "DELETE"})
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def _origin_parts(value):
|
| 21 |
+
raw = (value or "").strip()
|
| 22 |
+
if not raw or raw.lower() == "null":
|
| 23 |
+
return "", None
|
| 24 |
+
parsed = urlparse(raw)
|
| 25 |
+
hostname = (parsed.hostname or "").lower()
|
| 26 |
+
if not hostname:
|
| 27 |
+
return "", None
|
| 28 |
+
port = parsed.port
|
| 29 |
+
if port is None:
|
| 30 |
+
port = 443 if parsed.scheme == "https" else 80
|
| 31 |
+
return hostname, port
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _host_parts(host_header):
|
| 35 |
+
parsed = urlsplit("//" + (host_header or "").strip().lower())
|
| 36 |
+
hostname = (parsed.hostname or "").lower()
|
| 37 |
+
port = parsed.port
|
| 38 |
+
if port is None:
|
| 39 |
+
port = 80
|
| 40 |
+
return hostname, port
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def reject_csrf(request):
|
| 44 |
+
"""Return a 403 response if this mutating request is not same-origin."""
|
| 45 |
+
if request.method not in _MUTATING:
|
| 46 |
+
return None
|
| 47 |
+
|
| 48 |
+
site = (request.headers.get("Sec-Fetch-Site") or "").strip().lower()
|
| 49 |
+
if site == "cross-site":
|
| 50 |
+
return web.json_response(
|
| 51 |
+
{"error": "Cross-site request blocked."}, status=403)
|
| 52 |
+
|
| 53 |
+
host = (request.headers.get("Host") or "").strip()
|
| 54 |
+
hh, hp = _host_parts(host)
|
| 55 |
+
if not hh:
|
| 56 |
+
return web.json_response({"error": "Missing Host."}, status=403)
|
| 57 |
+
|
| 58 |
+
origin_raw = (request.headers.get("Origin") or "").strip()
|
| 59 |
+
oh, op = _origin_parts(origin_raw)
|
| 60 |
+
if not oh:
|
| 61 |
+
oh, op = _origin_parts(request.headers.get("Referer") or "")
|
| 62 |
+
if not oh:
|
| 63 |
+
return web.json_response(
|
| 64 |
+
{"error": "Missing Origin. Same-origin UI requests send it."},
|
| 65 |
+
status=403,
|
| 66 |
+
)
|
| 67 |
+
|
| 68 |
+
if oh != hh or op != hp:
|
| 69 |
+
return web.json_response(
|
| 70 |
+
{"error": "Origin does not match Host."}, status=403)
|
| 71 |
+
return None
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def require_same_origin(handler):
|
| 75 |
+
"""Decorator: reject CSRF-forgeable unauthenticated POST/DELETE."""
|
| 76 |
+
@wraps(handler)
|
| 77 |
+
async def wrapped(request):
|
| 78 |
+
blocked = reject_csrf(request)
|
| 79 |
+
if blocked is not None:
|
| 80 |
+
return blocked
|
| 81 |
+
return await handler(request)
|
| 82 |
+
return wrapped
|
custom_nodes/ComfyUI-H3-Motion-Context-main/example_workflows/MiniMax H3 - fl2va - ref2va.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
custom_nodes/ComfyUI-H3-Motion-Context-main/layout_contract.py
ADDED
|
@@ -0,0 +1,243 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Check that ComfyUI's H3 layout still means what this pack thinks it means.
|
| 2 |
+
|
| 3 |
+
Nothing here modifies ComfyUI. The pack used to wrap
|
| 4 |
+
`PackedLayout.__init__` because stock rejected any keyframe anchor other
|
| 5 |
+
than the first or last frame. ComfyUI 0.34.0 lifted that restriction, so the
|
| 6 |
+
node now just builds keyframe dicts and hands them over. No wrapper, no
|
| 7 |
+
ownership of a constructor, and no argument with other packs that patch
|
| 8 |
+
the same place.
|
| 9 |
+
|
| 10 |
+
What is left is a dependency on meaning rather than on structure. Stock
|
| 11 |
+
positions a keyframe at
|
| 12 |
+
|
| 13 |
+
cond_t = cursor + FRAME_RESCALE * kf["resolved_frame_index"]
|
| 14 |
+
|
| 15 |
+
where `cursor` is the target timeline's origin, already advanced past any
|
| 16 |
+
reference blocks. Two properties of that line matter to this pack and
|
| 17 |
+
neither is guaranteed by anything upstream:
|
| 18 |
+
|
| 19 |
+
fractional the audio window has to END at the join, so its index is
|
| 20 |
+
end_frame - rt / FRAME_RESCALE, which is not a whole number
|
| 21 |
+
negative that index is usually below zero, because the pinned audio
|
| 22 |
+
is a tail that played before this clip starts
|
| 23 |
+
|
| 24 |
+
Stock's own Add Guide node validates `frame_idx` to a non-negative integer
|
| 25 |
+
before it ever reaches the layout, so upstream has no test covering either
|
| 26 |
+
property. An int cast or a bounds check added later would not fail loudly
|
| 27 |
+
on its own: it would quietly move the pinned audio and put the join in the
|
| 28 |
+
wrong place.
|
| 29 |
+
|
| 30 |
+
So this runs once, before the first render, and proves the arithmetic
|
| 31 |
+
still holds. If it does not, the node refuses and says what moved. Same
|
| 32 |
+
policy as the patch self-test it replaces, without touching anything.
|
| 33 |
+
"""
|
| 34 |
+
|
| 35 |
+
import logging
|
| 36 |
+
|
| 37 |
+
import comfy.ldm.minimax.model as mm
|
| 38 |
+
|
| 39 |
+
_LOG = logging.getLogger("h3_motion_context")
|
| 40 |
+
|
| 41 |
+
_checked = None # None not yet run, True passed, str the failure
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
class _Stub:
|
| 45 |
+
"""Stands in for a latent. The layout reads only the shape."""
|
| 46 |
+
|
| 47 |
+
def __init__(self, shape):
|
| 48 |
+
self.shape = shape
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _origin(layout):
|
| 52 |
+
"""The coordinate the target clip starts at, read off the built layout.
|
| 53 |
+
|
| 54 |
+
The target video rows are always the last segment and their first row
|
| 55 |
+
sits exactly on the cursor, so this reads the origin rather than
|
| 56 |
+
recomputing stock's reference arithmetic. Everything else here is
|
| 57 |
+
expressed as a distance from it.
|
| 58 |
+
"""
|
| 59 |
+
a, b, kind = layout.segments[-1]
|
| 60 |
+
if kind != "video" or b <= a:
|
| 61 |
+
raise RuntimeError(
|
| 62 |
+
"expected the target video rows to be the last layout segment, "
|
| 63 |
+
"found %r spanning %d rows" % (kind, b - a))
|
| 64 |
+
return float(layout.position_ids[a, 0])
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _seg(layout, kind):
|
| 68 |
+
return [(a, b) for a, b, k in layout.segments if k == kind]
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _check():
|
| 72 |
+
text_len, latent_t, lh, lw, audio_t = 7, 7, 22, 38, 16
|
| 73 |
+
fr = mm.FRAME_RESCALE
|
| 74 |
+
|
| 75 |
+
if hasattr(mm.PackedLayout.__init__, "__wrapped__"):
|
| 76 |
+
pass # reported by ensure(); the checks below decide either way
|
| 77 |
+
|
| 78 |
+
def build(keyframes=None, refs=None):
|
| 79 |
+
return mm.PackedLayout(text_len, latent_t, lh, lw, audio_t,
|
| 80 |
+
keyframes=keyframes, refs=refs)
|
| 81 |
+
|
| 82 |
+
video = _Stub((1, 24, 1, lh, lw))
|
| 83 |
+
|
| 84 |
+
# 1. a whole-number anchor lands FRAME_RESCALE per pixel frame past the
|
| 85 |
+
# target origin. This is the line the whole pack is built on.
|
| 86 |
+
lay = build(keyframes=[{"resolved_frame_index": p, "latent": video}
|
| 87 |
+
for p in (0, 3)])
|
| 88 |
+
o = _origin(lay)
|
| 89 |
+
got = [float(lay.position_ids[a, 0]) - o for a, _ in _seg(lay, "cond")]
|
| 90 |
+
want = [0.0, fr * 3]
|
| 91 |
+
if len(got) != 2 or any(abs(g - w) > 1e-9 for g, w in zip(got, want)):
|
| 92 |
+
raise RuntimeError(
|
| 93 |
+
"keyframe anchors no longer sit at FRAME_RESCALE per pixel "
|
| 94 |
+
"frame past the target origin: got %s, expected %s" % (got, want))
|
| 95 |
+
|
| 96 |
+
# 2. a reference must not move an anchor relative to the target. Stock
|
| 97 |
+
# advances its cursor past the references before placing anchors, so
|
| 98 |
+
# the anchor-to-origin distance is unchanged.
|
| 99 |
+
lay = build(keyframes=[{"resolved_frame_index": 3, "latent": video}],
|
| 100 |
+
refs=[{"kind": "audio", "ref_audio_t": 8}])
|
| 101 |
+
a, _ = _seg(lay, "cond")[0]
|
| 102 |
+
gap = float(lay.position_ids[a, 0]) - _origin(lay)
|
| 103 |
+
if abs(gap - fr * 3) > 1e-9:
|
| 104 |
+
raise RuntimeError(
|
| 105 |
+
"a reference moved the anchors relative to the target: anchor "
|
| 106 |
+
"sits %.6f past the origin, expected %.6f. Stock no longer "
|
| 107 |
+
"compensates keyframes for reference blocks." % (gap, fr * 3))
|
| 108 |
+
|
| 109 |
+
# 3. the audio window: fractional AND negative index, window ending at
|
| 110 |
+
# a chosen target frame. This is the property with no upstream test.
|
| 111 |
+
# The numbers matter. An index is end_frame - rt / FRAME_RESCALE, and
|
| 112 |
+
# the node snaps end_frame so that FRAME_RESCALE * end_frame is a whole
|
| 113 |
+
# audio coordinate, which leaves the index whole whenever that
|
| 114 |
+
# coordinate and rt differ by a multiple of 5. Picking such a pair here
|
| 115 |
+
# would let an integer cast slip through the check, so this pair is
|
| 116 |
+
# deliberately not one of them.
|
| 117 |
+
rt, end_coord = 40, 8
|
| 118 |
+
end_frame = end_coord / fr
|
| 119 |
+
idx = end_frame - rt / fr
|
| 120 |
+
if idx >= 0 or idx == int(idx):
|
| 121 |
+
raise RuntimeError("the audio check is not exercising a fractional "
|
| 122 |
+
"negative index; its own numbers are wrong")
|
| 123 |
+
lay = build(keyframes=[{"resolved_frame_index": idx,
|
| 124 |
+
"audio_latent": _Stub((1, 32, 2, rt))}])
|
| 125 |
+
spans = _seg(lay, "cond_audio")
|
| 126 |
+
if len(spans) != 1:
|
| 127 |
+
raise RuntimeError(
|
| 128 |
+
"a keyframe carrying an audio latent produced %d cond_audio "
|
| 129 |
+
"segments, expected 1" % len(spans))
|
| 130 |
+
a, b = spans[0]
|
| 131 |
+
if b - a != 2 * rt:
|
| 132 |
+
raise RuntimeError(
|
| 133 |
+
"the pinned audio window has %d rows for %d latent steps, "
|
| 134 |
+
"expected %d (stereo, channel-major)" % (b - a, rt, 2 * rt))
|
| 135 |
+
o = _origin(lay)
|
| 136 |
+
t = lay.position_ids[a:b, 0]
|
| 137 |
+
start = float(t.min()) - o
|
| 138 |
+
end = start + float(rt)
|
| 139 |
+
if abs(end - fr * end_frame) > 1e-9:
|
| 140 |
+
raise RuntimeError(
|
| 141 |
+
"the pinned audio window ends %.6f past the target origin, "
|
| 142 |
+
"expected %.6f. A fractional or negative resolved_frame_index "
|
| 143 |
+
"is no longer taken literally, so pinned audio would land at "
|
| 144 |
+
"the wrong instant." % (end, fr * end_frame))
|
| 145 |
+
|
| 146 |
+
# 4. and it must reach BACKWARDS from there, not forwards. If upstream
|
| 147 |
+
# ever flips the window's anchor end this stays silent otherwise.
|
| 148 |
+
if start >= 0.0:
|
| 149 |
+
raise RuntimeError(
|
| 150 |
+
"the pinned audio window starts %.6f past the target origin; it "
|
| 151 |
+
"should start before it and end at the join" % start)
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def _wrapper_origin():
|
| 155 |
+
"""Who wrapped PackedLayout.__init__, if anyone. None if nobody has.
|
| 156 |
+
|
| 157 |
+
This matters because a wrapper's signature is not ComfyUI's. Every
|
| 158 |
+
known wrapper of this constructor is a copy of this pack's old layout
|
| 159 |
+
patch, vendored into another pack or left behind in a second folder,
|
| 160 |
+
and its signature carries `frame_count` on every ComfyUI version. Read
|
| 161 |
+
naively that looks exactly like ComfyUI 0.32, which is a confusing
|
| 162 |
+
thing to tell someone running 0.33.
|
| 163 |
+
|
| 164 |
+
So a wrapper is reported rather than interpreted, and the behavioural
|
| 165 |
+
checks below decide whether the layout still works. Usually it does:
|
| 166 |
+
the old patch passes anything without this pack's markers straight
|
| 167 |
+
through, and nothing built here carries them any more.
|
| 168 |
+
"""
|
| 169 |
+
init = getattr(mm.PackedLayout, "__init__", None)
|
| 170 |
+
if init is None:
|
| 171 |
+
return None
|
| 172 |
+
if getattr(init, "_h3_motion_context_layout_patch", False):
|
| 173 |
+
return "an older copy of this pack's layout patch"
|
| 174 |
+
if getattr(init, "__name__", "") == "_patched_init":
|
| 175 |
+
return "a copy of this pack's layout patch (a fork, or an older version)"
|
| 176 |
+
if hasattr(init, "__wrapped__"):
|
| 177 |
+
return "another pack (%s)" % getattr(init, "__module__", "unknown")
|
| 178 |
+
home = getattr(mm.PackedLayout, "__module__", None)
|
| 179 |
+
where = getattr(init, "__module__", None)
|
| 180 |
+
if home and where and where != home:
|
| 181 |
+
return "another pack (%s)" % where
|
| 182 |
+
return None
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def ensure(context=""):
|
| 186 |
+
"""Run the checks once. Raises with a usable message if they fail."""
|
| 187 |
+
global _checked
|
| 188 |
+
if _checked is True:
|
| 189 |
+
return
|
| 190 |
+
if isinstance(_checked, str):
|
| 191 |
+
raise RuntimeError(_checked)
|
| 192 |
+
|
| 193 |
+
if not hasattr(mm, "PackedLayout") or not hasattr(mm, "FRAME_RESCALE"):
|
| 194 |
+
_checked = ("h3_motion_context: ComfyUI's MiniMax H3 model module is "
|
| 195 |
+
"missing PackedLayout or FRAME_RESCALE. This pack cannot "
|
| 196 |
+
"run against it.")
|
| 197 |
+
raise RuntimeError(_checked)
|
| 198 |
+
|
| 199 |
+
import inspect
|
| 200 |
+
try:
|
| 201 |
+
params = inspect.signature(mm.PackedLayout.__init__).parameters
|
| 202 |
+
except (TypeError, ValueError):
|
| 203 |
+
params = {}
|
| 204 |
+
wrapped = _wrapper_origin()
|
| 205 |
+
if wrapped:
|
| 206 |
+
_LOG.warning(
|
| 207 |
+
"h3_motion_context: H3's layout constructor has been wrapped by "
|
| 208 |
+
"%s. This pack does not patch it and does not need it patched. "
|
| 209 |
+
"Checking whether anchors still land correctly; if they do, "
|
| 210 |
+
"nothing needs doing, but keeping only one H3 chaining pack "
|
| 211 |
+
"installed avoids surprises.", wrapped)
|
| 212 |
+
elif "frame_count" in params:
|
| 213 |
+
_checked = (
|
| 214 |
+
"h3_motion_context: this ComfyUI still has the older H3 layout, "
|
| 215 |
+
"which rejects keyframe anchors other than the first and last "
|
| 216 |
+
"frame. This version of the pack no longer works around that. "
|
| 217 |
+
"Update ComfyUI, or use pack version 0.3.1, which runs on both "
|
| 218 |
+
"layouts. (Detected from PackedLayout.__init__ still taking "
|
| 219 |
+
"frame_count, not from a version number.)")
|
| 220 |
+
raise RuntimeError(_checked)
|
| 221 |
+
|
| 222 |
+
try:
|
| 223 |
+
_check()
|
| 224 |
+
except Exception as exc:
|
| 225 |
+
_checked = (
|
| 226 |
+
"h3_motion_context: ComfyUI's H3 layout does not behave the way "
|
| 227 |
+
"this pack needs%s. %s. Refusing to run rather than rendering a "
|
| 228 |
+
"join at the wrong instant.%s"
|
| 229 |
+
% (" (%s)" % context if context else "", exc,
|
| 230 |
+
(" H3's layout constructor is wrapped by %s, which is the "
|
| 231 |
+
"first thing to rule out: disable it and try again."
|
| 232 |
+
% wrapped) if wrapped else
|
| 233 |
+
" This is an upstream ComfyUI change; please open an issue "
|
| 234 |
+
"with your ComfyUI version."))
|
| 235 |
+
raise RuntimeError(_checked) from exc
|
| 236 |
+
|
| 237 |
+
_checked = True
|
| 238 |
+
_LOG.info("h3_motion_context: ComfyUI H3 layout checks passed, anchors "
|
| 239 |
+
"and pinned audio will land where intended")
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
def is_checked():
|
| 243 |
+
return _checked is True
|
custom_nodes/ComfyUI-H3-Motion-Context-main/nodes.py
ADDED
|
@@ -0,0 +1,1201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pin previous-clip motion at the head of an H3 clip.
|
| 2 |
+
|
| 3 |
+
Wire it between a stock H3 conditioning node and the sampler:
|
| 4 |
+
|
| 5 |
+
MiniMaxH3ImageToVideo / MiniMaxH3ReferenceToVideo (or the t2v path)
|
| 6 |
+
-> H3 Motion Context
|
| 7 |
+
-> guider / sampler
|
| 8 |
+
|
| 9 |
+
Two axes to test, both cheap.
|
| 10 |
+
|
| 11 |
+
encode_mode
|
| 12 |
+
frames one VAE call per frame, each pinned as its own cond block. The
|
| 13 |
+
model sees N snapshots at N instants.
|
| 14 |
+
video one VAE call for the whole run. The H3 video VAE has latent_dim
|
| 15 |
+
3, so it reads the batch axis as time and compresses the run
|
| 16 |
+
into fewer latent steps (5 pixel frames -> 2 steps, 22 -> 7).
|
| 17 |
+
Each step becomes one cond block, so the motion between frames
|
| 18 |
+
lives inside the latent instead of being implied across separate
|
| 19 |
+
stills. Far fewer rows and one VAE load.
|
| 20 |
+
|
| 21 |
+
anchor_mode
|
| 22 |
+
head pinned frames occupy indices 0..N-1 of the delivered timeline.
|
| 23 |
+
They come back in the output, so trim that many frames off the
|
| 24 |
+
front before concatenating.
|
| 25 |
+
before pinned frames sit at negative indices, ending at -1, so
|
| 26 |
+
delivered frame 0 continues from them and nothing is wasted.
|
| 27 |
+
Their time coordinates land below text_len, which is the range
|
| 28 |
+
the text rows occupy. Whether that collision matters is exactly
|
| 29 |
+
what this mode is asking.
|
| 30 |
+
"""
|
| 31 |
+
|
| 32 |
+
import gc
|
| 33 |
+
import logging
|
| 34 |
+
import os
|
| 35 |
+
import re
|
| 36 |
+
|
| 37 |
+
import comfy.utils
|
| 38 |
+
import folder_paths
|
| 39 |
+
import node_helpers
|
| 40 |
+
import torch
|
| 41 |
+
|
| 42 |
+
try:
|
| 43 |
+
from safetensors.torch import load_file as _st_load, save_file as _st_save
|
| 44 |
+
except ImportError: # ComfyUI always ships safetensors; belt and braces
|
| 45 |
+
_st_load = _st_save = None
|
| 46 |
+
|
| 47 |
+
from .layout_contract import ensure as _ensure_layout_contract
|
| 48 |
+
from .layout_contract import is_checked as _layout_checked
|
| 49 |
+
|
| 50 |
+
try:
|
| 51 |
+
import torchaudio
|
| 52 |
+
except ImportError:
|
| 53 |
+
torchaudio = None
|
| 54 |
+
|
| 55 |
+
_LOG = logging.getLogger("h3_motion_context")
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _ensure_layout_ok():
|
| 59 |
+
"""Prove ComfyUI still places anchors the way this pack needs, once.
|
| 60 |
+
|
| 61 |
+
This used to install two runtime patches. ComfyUI 0.33 does natively
|
| 62 |
+
what they existed to do, so the node now builds plain keyframe dicts
|
| 63 |
+
and only has to check that the arithmetic behind them still holds. See
|
| 64 |
+
layout_contract.py for what is checked and why it is not free.
|
| 65 |
+
|
| 66 |
+
Run on first use rather than at import, same as the patches were: the
|
| 67 |
+
pack sitting in custom_nodes should change nothing at all until you
|
| 68 |
+
actually chain a clip. The cost is that a failure shows up on the
|
| 69 |
+
first render instead of in the startup log.
|
| 70 |
+
"""
|
| 71 |
+
_ensure_layout_contract("pinning a clip")
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
FRAME_PER_TOKEN = (1, 4, 4, 4, 4)
|
| 75 |
+
FPS = 24 # H3's native rate; audio latents run at 40 Hz, hence FRAME_RESCALE 5/3
|
| 76 |
+
FRAME_RESCALE = 5.0 / 3.0
|
| 77 |
+
AUDIO_HZ = 40.0
|
| 78 |
+
|
| 79 |
+
# Run lengths the video VAE's downscale formula max(1, (n - 5) // 17 * 5 + 2)
|
| 80 |
+
# actually distinguishes. Anything between two grid points encodes to the same
|
| 81 |
+
# number of latent steps as the lower one, but the steps then cover the FIRST
|
| 82 |
+
# `covered` frames of the input rather than the last: encoding 10 frames yields
|
| 83 |
+
# the same 2 steps as encoding 5, representing frames [-10..-6] of the source
|
| 84 |
+
# clip instead of [-5..-1]. The pinned run would end five frames early and the
|
| 85 |
+
# delivered clip would continue from the wrong instant. So off-grid requests
|
| 86 |
+
# are snapped DOWN before slicing, keeping content and coverage in agreement.
|
| 87 |
+
# The grid is 17m+5 and continues upward; the node only offers up to 56,
|
| 88 |
+
# but the snap-down logic knows the higher points so an out-of-range
|
| 89 |
+
# request lands on the nearest real one instead of being clamped to 39.
|
| 90 |
+
VIDEO_RUN_GRID = (124, 107, 90, 73, 56, 39, 22, 5, 1)
|
| 91 |
+
|
| 92 |
+
# Settings that used to be widgets. Each had exactly one right answer, so
|
| 93 |
+
# offering the wrong one was noise. The losing branches are still in the
|
| 94 |
+
# code below: change a constant here to reproduce the failure they cause.
|
| 95 |
+
#
|
| 96 |
+
# ENCODE_MODE "video" encodes the pinned run in one VAE call, so the
|
| 97 |
+
# motion lives inside the latent. "frames" encodes each
|
| 98 |
+
# frame as its own still, costs twice the rows and left a
|
| 99 |
+
# visible seam in testing.
|
| 100 |
+
# ANCHOR_MODE "head" pins the run at the start of the clip, where the
|
| 101 |
+
# Trim node removes it. "before" places it at negative
|
| 102 |
+
# time so nothing needs trimming, but the coordinates
|
| 103 |
+
# collide with the text rows, which weakens the anchors
|
| 104 |
+
# and darkens the output.
|
| 105 |
+
# AUDIO_MODE "timeline" puts the pinned audio on this clip's own
|
| 106 |
+
# timeline so the model continues it. "ref" is the stock
|
| 107 |
+
# placement, which the model imitates instead: similar
|
| 108 |
+
# music, not the same recording, plus a tick at the join.
|
| 109 |
+
# CROP only ever applied when an aspect ratio changed between
|
| 110 |
+
# clips, which the resolution check now refuses outright.
|
| 111 |
+
ENCODE_MODE = "video"
|
| 112 |
+
ANCHOR_MODE = "head"
|
| 113 |
+
AUDIO_MODE = "timeline"
|
| 114 |
+
CROP = "disabled"
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def _pixel_frames(latent_t):
|
| 118 |
+
"""Pixel frames covered by latent_t latent steps."""
|
| 119 |
+
return sum(FRAME_PER_TOKEN[k % 5] for k in range(latent_t))
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def _step_offsets(latent_t):
|
| 123 |
+
"""Pixel-frame index at which each latent step begins."""
|
| 124 |
+
out, acc = [], 0
|
| 125 |
+
for k in range(latent_t):
|
| 126 |
+
out.append(acc)
|
| 127 |
+
acc += FRAME_PER_TOKEN[k % 5]
|
| 128 |
+
return out
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def _resize(image, width, height, crop):
|
| 132 |
+
# image [B, H, W, C] -> [B, height, width, 3]; matches the stock helper
|
| 133 |
+
samples = image[..., :3].movedim(-1, 1)
|
| 134 |
+
samples = comfy.utils.common_upscale(samples, width, height, "lanczos", crop)
|
| 135 |
+
return samples.movedim(1, -1)
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def _encode_tail_audio(audio_vae, audio, seconds):
|
| 139 |
+
"""Encode the last `seconds` of a clip's audio with the H3 audio VAE.
|
| 140 |
+
|
| 141 |
+
Returns ([1, 32, 2, T] latent, T) where T counts 40 Hz latent steps,
|
| 142 |
+
matching what the layout calls ref_audio_t.
|
| 143 |
+
"""
|
| 144 |
+
waveform = audio["waveform"] # [B, C, L]
|
| 145 |
+
sr = int(audio["sample_rate"])
|
| 146 |
+
vae_sr = int(getattr(audio_vae, "audio_sample_rate", 32000))
|
| 147 |
+
if sr != vae_sr:
|
| 148 |
+
if torchaudio is None:
|
| 149 |
+
raise RuntimeError(
|
| 150 |
+
"h3_motion_context: context_audio is %d Hz but the VAE wants %d Hz "
|
| 151 |
+
"and torchaudio is not available to resample." % (sr, vae_sr))
|
| 152 |
+
waveform = torchaudio.functional.resample(waveform, sr, vae_sr)
|
| 153 |
+
want = int(round(seconds * vae_sr))
|
| 154 |
+
have = int(waveform.shape[-1])
|
| 155 |
+
if have < want:
|
| 156 |
+
_LOG.warning("h3_motion_context: context_audio is %.3fs, shorter than the "
|
| 157 |
+
"%.3fs of pinned video. Pinning what there is.",
|
| 158 |
+
have / vae_sr, seconds)
|
| 159 |
+
else:
|
| 160 |
+
waveform = waveform[..., have - want:]
|
| 161 |
+
z = audio_vae.encode(waveform[:1].movedim(1, -1)) # [1, 32, 2, T]
|
| 162 |
+
return z, int(z.shape[-1])
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def _streams_from_latent(latent):
|
| 166 |
+
"""Unpack an H3 AV latent into its contained streams.
|
| 167 |
+
|
| 168 |
+
NestedTensor.__getitem__ broadcasts the index into every contained
|
| 169 |
+
tensor rather than selecting one, so samples[0] would strip the batch
|
| 170 |
+
dimension off both streams. unbind() returns the pair.
|
| 171 |
+
"""
|
| 172 |
+
samples = latent["samples"]
|
| 173 |
+
if hasattr(samples, "unbind"):
|
| 174 |
+
parts = list(samples.unbind())
|
| 175 |
+
elif isinstance(samples, (tuple, list)):
|
| 176 |
+
parts = list(samples)
|
| 177 |
+
else:
|
| 178 |
+
raise ValueError(
|
| 179 |
+
"h3_motion_context: expected a MiniMax H3 AV latent (a nested "
|
| 180 |
+
"video/audio pair), got %r" % type(samples))
|
| 181 |
+
if not parts:
|
| 182 |
+
raise ValueError("h3_motion_context: AV latent contains no streams")
|
| 183 |
+
return parts
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def _video_from_latent(latent):
|
| 187 |
+
"""Pull the video stream out of an H3 AV latent."""
|
| 188 |
+
video = _streams_from_latent(latent)[0]
|
| 189 |
+
if video.ndim == 4: # unbatched [C,T,H,W]
|
| 190 |
+
video = video.unsqueeze(0)
|
| 191 |
+
if video.ndim != 5:
|
| 192 |
+
raise ValueError("h3_motion_context: expected video latent [B,C,T,H,W], "
|
| 193 |
+
"got shape %s" % (tuple(video.shape),))
|
| 194 |
+
return video
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def _steps_for_frames(n):
|
| 198 |
+
"""Latent steps covering exactly n pixel frames from cycle position 0.
|
| 199 |
+
|
| 200 |
+
Returns None when no whole number of steps covers n. The video VAE's
|
| 201 |
+
steps alternate 1, 4, 4, 4, 4 pixel frames, so only certain totals are
|
| 202 |
+
reachable: 1, 5, 9, ... and of the windows this node offers, 5, 22, 39
|
| 203 |
+
and 56 land on 2, 7, 12 and 17 steps. The 1-frame window does not,
|
| 204 |
+
because the last step of a clip spans 4 frames, not 1.
|
| 205 |
+
"""
|
| 206 |
+
k, covered = 0, 0
|
| 207 |
+
while covered < n:
|
| 208 |
+
covered += FRAME_PER_TOKEN[k % 5]
|
| 209 |
+
k += 1
|
| 210 |
+
return k if covered == n else None
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def _video_tail_from_latent(latent, n):
|
| 214 |
+
"""Slice the last n pixel frames of video straight out of a generated
|
| 215 |
+
H3 latent, skipping the h264 decode and the VAE encode.
|
| 216 |
+
|
| 217 |
+
Returns (blocks, offsets, covered) in the same shape the encode path
|
| 218 |
+
produces, so everything downstream is unchanged.
|
| 219 |
+
|
| 220 |
+
This is only sound because the tail window always starts at cycle
|
| 221 |
+
position 0. A clip is 17g+5 frames, which is 5g+2 latent steps; the
|
| 222 |
+
windows are 2, 7, 12 and 17 steps; and 5g+2 minus any of those is a
|
| 223 |
+
multiple of 5. So the sliced run has the same 1, 4, 4, 4, 4 phase as a
|
| 224 |
+
freshly encoded one and _step_offsets applies unchanged. Asserted
|
| 225 |
+
below rather than assumed, because if it ever stopped holding the
|
| 226 |
+
pinned content would silently disagree with the positions written for
|
| 227 |
+
it and the join would land at the wrong instant.
|
| 228 |
+
"""
|
| 229 |
+
video = _video_from_latent(latent)
|
| 230 |
+
total = int(video.shape[2])
|
| 231 |
+
steps = _steps_for_frames(n)
|
| 232 |
+
if steps is None:
|
| 233 |
+
raise ValueError(
|
| 234 |
+
"h3_motion_context: a %d frame window is not a whole number of "
|
| 235 |
+
"latent steps, so it cannot be sliced from a latent. Use 5, 22, "
|
| 236 |
+
"39 or 56, or unwire context_latent to encode pixels." % n)
|
| 237 |
+
if steps > total:
|
| 238 |
+
raise ValueError(
|
| 239 |
+
"h3_motion_context: asked for %d latent steps, context_latent "
|
| 240 |
+
"has %d." % (steps, total))
|
| 241 |
+
start = total - steps
|
| 242 |
+
if start % 5 != 0:
|
| 243 |
+
raise RuntimeError(
|
| 244 |
+
"h3_motion_context: the %d step tail of a %d step latent starts "
|
| 245 |
+
"at cycle position %d, not 0, so its frame spans would not match "
|
| 246 |
+
"the positions written for them. Clip lengths are meant to make "
|
| 247 |
+
"this impossible; refusing rather than rendering a shifted join."
|
| 248 |
+
% (steps, total, start % 5))
|
| 249 |
+
covered = _pixel_frames(steps)
|
| 250 |
+
if covered != n:
|
| 251 |
+
raise RuntimeError(
|
| 252 |
+
"h3_motion_context: %d steps cover %d frames, expected %d."
|
| 253 |
+
% (steps, covered, n))
|
| 254 |
+
blocks = [video[:1, :, start + k:start + k + 1].clone()
|
| 255 |
+
for k in range(steps)]
|
| 256 |
+
return blocks, _step_offsets(steps), covered
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
def _audio_tail_from_latent(latent, a_frames):
|
| 260 |
+
"""Slice the last `a_frames` worth of audio steps straight out of a
|
| 261 |
+
generated H3 latent, skipping the decode -> re-encode round trip.
|
| 262 |
+
|
| 263 |
+
Returns (tail latent [1, C, 2, rt], rt, overhang) where rt counts
|
| 264 |
+
40 Hz latent steps and overhang is the signed fraction of a step by
|
| 265 |
+
which the clip's audio grid overshoots its last pixel frame.
|
| 266 |
+
|
| 267 |
+
H3 rounds the audio grid to the NEAREST step, not up, so overhang is
|
| 268 |
+
negative for a third of legal clip lengths. 5/3 of a frame count
|
| 269 |
+
lands on .0, .333 or .667 and never on .5, so there are exactly three
|
| 270 |
+
cases:
|
| 271 |
+
|
| 272 |
+
frames % 3 == 0 243 wants 405.00, allocates 405, overhang 0
|
| 273 |
+
frames % 3 == 1 124 wants 206.67, allocates 207, overhang +1/3
|
| 274 |
+
frames % 3 == 2 260 wants 433.33, allocates 433, overhang -1/3
|
| 275 |
+
|
| 276 |
+
A positive overhang means the latent's final step reaches past the
|
| 277 |
+
last frame, a negative one means it stops short. Either way the
|
| 278 |
+
caller compensates the placement with it, so the pinned content lands
|
| 279 |
+
where its samples actually sit. The decoded-audio path never sees
|
| 280 |
+
this because match_tail cuts at the frame.
|
| 281 |
+
"""
|
| 282 |
+
parts = _streams_from_latent(latent)
|
| 283 |
+
if len(parts) < 2:
|
| 284 |
+
raise ValueError(
|
| 285 |
+
"h3_motion_context: context_latent has no audio stream. Wire the "
|
| 286 |
+
"sampler output of an H3 AV graph, not a video-only latent.")
|
| 287 |
+
video, audio = parts[0], parts[1]
|
| 288 |
+
if video.ndim == 4:
|
| 289 |
+
video = video.unsqueeze(0)
|
| 290 |
+
if audio.ndim == 3: # unbatched [C,2,T]
|
| 291 |
+
audio = audio.unsqueeze(0)
|
| 292 |
+
if audio.ndim != 4:
|
| 293 |
+
raise ValueError("h3_motion_context: expected audio latent [B,C,2,T], "
|
| 294 |
+
"got shape %s" % (tuple(audio.shape),))
|
| 295 |
+
total_t = int(audio.shape[-1])
|
| 296 |
+
frames = _pixel_frames(int(video.shape[2]))
|
| 297 |
+
overhang = total_t - FRAME_RESCALE * frames
|
| 298 |
+
# legal values are exactly 0, +1/3 and -1/3; the band is the widest
|
| 299 |
+
# one that admits all three and still rejects a grid that is out by a
|
| 300 |
+
# whole step or more
|
| 301 |
+
if not (-0.5 < overhang < 0.5):
|
| 302 |
+
_LOG.warning(
|
| 303 |
+
"h3_motion_context: context_latent audio grid is unexpected "
|
| 304 |
+
"(%d steps for %d frames); assuming no overhang.", total_t, frames)
|
| 305 |
+
overhang = 0.0
|
| 306 |
+
rt = int(round(a_frames / float(FPS) * AUDIO_HZ))
|
| 307 |
+
if rt > total_t:
|
| 308 |
+
_LOG.warning("h3_motion_context: asked for %d audio steps, the latent "
|
| 309 |
+
"has %d. Pinning all of it.", rt, total_t)
|
| 310 |
+
rt = total_t
|
| 311 |
+
if rt < 1:
|
| 312 |
+
raise ValueError("h3_motion_context: audio window is empty")
|
| 313 |
+
tail = audio[:1, ..., total_t - rt:].clone()
|
| 314 |
+
return tail, rt, float(overhang)
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
class MiniMaxH3MotionContext:
|
| 318 |
+
@classmethod
|
| 319 |
+
def INPUT_TYPES(cls):
|
| 320 |
+
return {
|
| 321 |
+
"required": {
|
| 322 |
+
"conditioning": ("CONDITIONING",),
|
| 323 |
+
"vae": ("VAE",),
|
| 324 |
+
"latent": ("LATENT",),
|
| 325 |
+
"context_length": (["22", "5", "39", "56"], {
|
| 326 |
+
"default": "22",
|
| 327 |
+
"tooltip": "Frames of the previous clip's picture to "
|
| 328 |
+
"carry over. Only these lengths are whole "
|
| 329 |
+
"numbers of latent steps, so only these are "
|
| 330 |
+
"offered. 5 is just barely fluid, 22 is "
|
| 331 |
+
"nearly seamless. Longer windows pin more "
|
| 332 |
+
"motion but come off the front of the "
|
| 333 |
+
"delivered clip, so 56 spends 2.3 seconds of "
|
| 334 |
+
"the render on frames you throw away."}),
|
| 335 |
+
"audio_context_length": ("INT", {
|
| 336 |
+
"default": 24, "min": 0, "max": 240,
|
| 337 |
+
"tooltip": "Frames of tail audio to pin, independent of "
|
| 338 |
+
"the picture window. 0 follows it. The window "
|
| 339 |
+
"is END-aligned with the pinned video, so "
|
| 340 |
+
"this only controls how far back the sound "
|
| 341 |
+
"reaches. Multiples of 3 land exactly on the "
|
| 342 |
+
"40 Hz audio grid and multiples of 24 are "
|
| 343 |
+
"whole seconds: 24 pins the last second. "
|
| 344 |
+
"Off-grid values are widened to the nearest "
|
| 345 |
+
"whole step."}),
|
| 346 |
+
},
|
| 347 |
+
"optional": {
|
| 348 |
+
"context_frames": ("IMAGE", {
|
| 349 |
+
"tooltip": "Decoded frames of the previous clip. Used "
|
| 350 |
+
"when no context_latent is wired. When one "
|
| 351 |
+
"is, the picture comes from it instead and "
|
| 352 |
+
"this is ignored."}),
|
| 353 |
+
"context_latent": ("LATENT", {
|
| 354 |
+
"tooltip": "Previous clip's SAMPLER OUTPUT latent (the "
|
| 355 |
+
"same one you wire into the decode nodes). "
|
| 356 |
+
"Supplies both picture and sound, sliced "
|
| 357 |
+
"straight out, skipping the decode and "
|
| 358 |
+
"re-encode that cost a little quality at "
|
| 359 |
+
"every link of a chain. Must be the same "
|
| 360 |
+
"resolution as the clip being generated."}),
|
| 361 |
+
"audio_vae": ("VAE", {
|
| 362 |
+
"tooltip": "H3 audio VAE. Supply with context_audio to "
|
| 363 |
+
"carry the previous clip's tail sound across "
|
| 364 |
+
"the join. Not needed when context_latent is "
|
| 365 |
+
"wired."}),
|
| 366 |
+
"context_audio": ("AUDIO", {
|
| 367 |
+
"tooltip": "Audio of the previous clip. The tail "
|
| 368 |
+
"matching the pinned frames is encoded and "
|
| 369 |
+
"pinned alongside them. Ignored when "
|
| 370 |
+
"context_latent is wired."}),
|
| 371 |
+
},
|
| 372 |
+
}
|
| 373 |
+
|
| 374 |
+
RETURN_TYPES = ("CONDITIONING", "INT")
|
| 375 |
+
RETURN_NAMES = ("conditioning", "trim_frames")
|
| 376 |
+
FUNCTION = "apply"
|
| 377 |
+
CATEGORY = "conditioning/minimax"
|
| 378 |
+
DESCRIPTION = ("Pin a run of consecutive frames from a previous clip as "
|
| 379 |
+
"never-denoised conditioning rows, so the model reads real "
|
| 380 |
+
"motion instead of guessing it from a single still. With "
|
| 381 |
+
"context_latent wired, both picture and sound are sliced "
|
| 382 |
+
"from the previous clip's latent, skipping the decode and "
|
| 383 |
+
"re-encode that cost a little quality at every link. A "
|
| 384 |
+
"last_frame anchor on the incoming conditioning is kept "
|
| 385 |
+
"and pinned alongside the head.")
|
| 386 |
+
|
| 387 |
+
def apply(self, conditioning, vae, latent, context_length,
|
| 388 |
+
audio_context_length=24, context_frames=None,
|
| 389 |
+
context_latent=None, audio_vae=None, context_audio=None):
|
| 390 |
+
if context_latent is None and context_frames is None:
|
| 391 |
+
return (conditioning, 0)
|
| 392 |
+
encode_mode, anchor_mode = ENCODE_MODE, ANCHOR_MODE
|
| 393 |
+
audio_mode, crop = AUDIO_MODE, CROP
|
| 394 |
+
context_length = int(context_length)
|
| 395 |
+
_ensure_layout_ok()
|
| 396 |
+
|
| 397 |
+
video = _video_from_latent(latent)
|
| 398 |
+
latent_t = int(video.shape[2])
|
| 399 |
+
width = int(video.shape[4]) * 16
|
| 400 |
+
height = int(video.shape[3]) * 16
|
| 401 |
+
frame_count = _pixel_frames(latent_t)
|
| 402 |
+
|
| 403 |
+
# Decide where the pinned VIDEO comes from before anything else,
|
| 404 |
+
# because it decides how many frames are even available. Slicing it
|
| 405 |
+
# out of the previous clip's latent removes an h264 decode and a
|
| 406 |
+
# VAE encode from the path, and the blocks come out bit-identical
|
| 407 |
+
# to what the model produced rather than a reconstruction of it.
|
| 408 |
+
# A wired latent supplies the picture as well as the sound: the
|
| 409 |
+
# pinned blocks are then exactly the steps the model produced,
|
| 410 |
+
# with no h264 decode, no resize and no VAE round trip to shift
|
| 411 |
+
# colour or contrast. Frames are the path when no latent is wired.
|
| 412 |
+
if context_latent is not None:
|
| 413 |
+
src_video = _video_from_latent(context_latent)
|
| 414 |
+
src_w = int(src_video.shape[4]) * 16
|
| 415 |
+
src_h = int(src_video.shape[3]) * 16
|
| 416 |
+
if src_w != width or src_h != height:
|
| 417 |
+
# a latent cannot be resized. Falling back to frames here
|
| 418 |
+
# would quietly take the lossy path on a graph the user
|
| 419 |
+
# thinks is fine, and a resolution change mid-chain is
|
| 420 |
+
# nearly always a mistake, so say so instead.
|
| 421 |
+
raise ValueError(
|
| 422 |
+
"h3_motion_context: context_latent is %dx%d but this "
|
| 423 |
+
"clip is %dx%d. A latent cannot be resized, so the "
|
| 424 |
+
"previous clip has to be regenerated at this "
|
| 425 |
+
"resolution, or the chain restarted here."
|
| 426 |
+
% (src_w, src_h, width, height))
|
| 427 |
+
if int(src_video.shape[1]) != int(video.shape[1]):
|
| 428 |
+
raise ValueError(
|
| 429 |
+
"h3_motion_context: context_latent has %d channels, "
|
| 430 |
+
"this clip has %d. That is not an H3 video latent from "
|
| 431 |
+
"the same model."
|
| 432 |
+
% (int(src_video.shape[1]), int(video.shape[1])))
|
| 433 |
+
available = _pixel_frames(int(src_video.shape[2]))
|
| 434 |
+
video_src = "latent"
|
| 435 |
+
else:
|
| 436 |
+
available = int(context_frames.shape[0])
|
| 437 |
+
video_src = "pixels"
|
| 438 |
+
|
| 439 |
+
n = min(int(context_length), available)
|
| 440 |
+
if n < 1:
|
| 441 |
+
raise ValueError("h3_motion_context: no frames available to pin")
|
| 442 |
+
if n < context_length:
|
| 443 |
+
_LOG.warning("h3_motion_context: only %d frames available, pinning %d",
|
| 444 |
+
available, n)
|
| 445 |
+
|
| 446 |
+
if encode_mode == "video":
|
| 447 |
+
# snap down to the VAE grid BEFORE slicing, so the frames encoded
|
| 448 |
+
# are exactly the frames the latent steps will cover (see
|
| 449 |
+
# VIDEO_RUN_GRID). Slicing the last n and letting the VAE keep the
|
| 450 |
+
# first `covered` of them would pin a run ending before the clip
|
| 451 |
+
# does, and the join would jump by the difference.
|
| 452 |
+
run = next(g for g in VIDEO_RUN_GRID if g <= n)
|
| 453 |
+
if run != n:
|
| 454 |
+
_LOG.warning(
|
| 455 |
+
"h3_motion_context: %d frames is off the VAE grid; pinning "
|
| 456 |
+
"the last %d instead (usable runs: 1, 5, 22, 39, 56)", n, run)
|
| 457 |
+
n = run
|
| 458 |
+
|
| 459 |
+
if n >= frame_count:
|
| 460 |
+
raise ValueError(
|
| 461 |
+
"h3_motion_context: asked to pin %d frames into a %d frame clip. "
|
| 462 |
+
"The pinned run must be a small fraction of the timeline."
|
| 463 |
+
% (n, frame_count))
|
| 464 |
+
|
| 465 |
+
if video_src == "latent" and _steps_for_frames(n) is None:
|
| 466 |
+
# every window the node offers is a whole number of steps, so
|
| 467 |
+
# reaching this means the grid moved underneath us
|
| 468 |
+
raise RuntimeError(
|
| 469 |
+
"h3_motion_context: a %d frame window is not a whole number "
|
| 470 |
+
"of latent steps. VIDEO_RUN_GRID no longer matches the "
|
| 471 |
+
"VAE; refusing rather than rendering a shifted join." % n)
|
| 472 |
+
|
| 473 |
+
if video_src == "latent":
|
| 474 |
+
blocks, offsets, covered = _video_tail_from_latent(
|
| 475 |
+
context_latent, n)
|
| 476 |
+
span = covered
|
| 477 |
+
else:
|
| 478 |
+
# the LAST n frames of the incoming clip become the pinned run
|
| 479 |
+
tail = _resize(context_frames[available - n:], width, height, crop)
|
| 480 |
+
|
| 481 |
+
if video_src == "pixels" and encode_mode == "video":
|
| 482 |
+
# one call; the VAE reads the batch axis as time and compresses
|
| 483 |
+
enc = vae.encode(tail)
|
| 484 |
+
if getattr(enc, "ndim", 0) != 5:
|
| 485 |
+
raise ValueError(
|
| 486 |
+
"h3_motion_context: video-mode encode returned shape %s, "
|
| 487 |
+
"expected [B,C,T,H,W]. Try encode_mode=frames."
|
| 488 |
+
% (tuple(getattr(enc, "shape", ())),))
|
| 489 |
+
steps = int(enc.shape[2])
|
| 490 |
+
offsets = _step_offsets(steps)
|
| 491 |
+
covered = _pixel_frames(steps)
|
| 492 |
+
if covered != n:
|
| 493 |
+
# n was snapped to the grid above, so a mismatch here means
|
| 494 |
+
# the VAE's downscale formula changed underneath us and the
|
| 495 |
+
# pinned content no longer lines up with the positions we
|
| 496 |
+
# would write. Refuse rather than render a shifted join.
|
| 497 |
+
raise RuntimeError(
|
| 498 |
+
"h3_motion_context: %d frames encoded to %d latent steps "
|
| 499 |
+
"covering %d frames; the VAE grid no longer matches "
|
| 500 |
+
"VIDEO_RUN_GRID. Upstream VAE change, refusing to run."
|
| 501 |
+
% (n, steps, covered))
|
| 502 |
+
blocks = [enc[:, :, k:k + 1] for k in range(steps)]
|
| 503 |
+
span = covered
|
| 504 |
+
elif video_src == "pixels":
|
| 505 |
+
blocks, offsets = [], []
|
| 506 |
+
for i in range(n):
|
| 507 |
+
blocks.append(vae.encode(tail[i:i + 1]))
|
| 508 |
+
offsets.append(i)
|
| 509 |
+
span = n
|
| 510 |
+
|
| 511 |
+
if anchor_mode == "before":
|
| 512 |
+
indices = [o - span for o in offsets]
|
| 513 |
+
else:
|
| 514 |
+
indices = list(offsets)
|
| 515 |
+
|
| 516 |
+
keyframes = []
|
| 517 |
+
for p, blk in zip(indices, blocks):
|
| 518 |
+
# Straight to stock. `resolved_frame_index` is the real pixel
|
| 519 |
+
# frame the block sits at, which ComfyUI 0.33 accepts for any
|
| 520 |
+
# value, so there is nothing to smuggle and nothing to rewrite
|
| 521 |
+
# afterwards. One keyframe per latent step rather than a single
|
| 522 |
+
# multi-step block: `indices` already carries each step's real
|
| 523 |
+
# offset, which keeps the encode_mode=frames path (offsets
|
| 524 |
+
# 0..n-1, one still each) on exactly the same code. Stock lays
|
| 525 |
+
# a 1-step latent at cursor + FRAME_RESCALE * index either way,
|
| 526 |
+
# so the two forms produce identical rows.
|
| 527 |
+
keyframes.append({
|
| 528 |
+
"resolved_frame_index": p,
|
| 529 |
+
"latent": blk,
|
| 530 |
+
})
|
| 531 |
+
|
| 532 |
+
ref_audio_t = 0
|
| 533 |
+
audio_ref = None
|
| 534 |
+
audio_kf = None
|
| 535 |
+
audio_end_frame = None
|
| 536 |
+
a_frames = 0
|
| 537 |
+
audio_src = "off"
|
| 538 |
+
if context_latent is not None or context_audio is not None:
|
| 539 |
+
# the audio window is independent of the video one: audio cond
|
| 540 |
+
# rows cost rows but never cost delivered frames
|
| 541 |
+
a_frames = int(audio_context_length) or span
|
| 542 |
+
if context_latent is not None:
|
| 543 |
+
if context_audio is not None:
|
| 544 |
+
_LOG.info("h3_motion_context: both context_latent and "
|
| 545 |
+
"context_audio wired; using the latent (skips "
|
| 546 |
+
"one VAE round trip).")
|
| 547 |
+
audio_latent, ref_audio_t, overhang = _audio_tail_from_latent(
|
| 548 |
+
context_latent, a_frames)
|
| 549 |
+
audio_src = "latent"
|
| 550 |
+
else:
|
| 551 |
+
if audio_vae is None:
|
| 552 |
+
raise ValueError(
|
| 553 |
+
"h3_motion_context: context_audio supplied without "
|
| 554 |
+
"audio_vae. Wire the H3 audio VAE, or wire "
|
| 555 |
+
"context_latent instead.")
|
| 556 |
+
audio_latent, ref_audio_t = _encode_tail_audio(
|
| 557 |
+
audio_vae, context_audio, a_frames / float(FPS))
|
| 558 |
+
overhang = 0.0 # decoded audio was match_tail-cut at the frame
|
| 559 |
+
audio_src = "vae"
|
| 560 |
+
if audio_mode == "timeline":
|
| 561 |
+
# end-align the audio window with the pinned video: both are
|
| 562 |
+
# the tail of clip A, so both must end at the same instant
|
| 563 |
+
# of the new timeline, frame `span` in head mode (where
|
| 564 |
+
# A's last frame sits), frame 0 in before mode. On the
|
| 565 |
+
# latent path the sliced content overshoots A's last
|
| 566 |
+
# frame by `overhang` of a step, signed, because H3
|
| 567 |
+
# rounds its audio grid to the nearest step and so falls
|
| 568 |
+
# short as often as it reaches past. The end coordinate
|
| 569 |
+
# moves by exactly that much, and a keyframe index is a
|
| 570 |
+
# plain multiplier so it takes a fractional frame.
|
| 571 |
+
end_frame = float(span if anchor_mode == "head" else 0)
|
| 572 |
+
end_frame += overhang / FRAME_RESCALE
|
| 573 |
+
# then snap the window onto the target's own audio grid.
|
| 574 |
+
# The end coordinate is FRAME_RESCALE * end_frame, and
|
| 575 |
+
# FRAME_RESCALE is 5/3, so unless that product happens to
|
| 576 |
+
# be a whole number the pinned rows land between the
|
| 577 |
+
# integer coordinates the target's audio rows occupy. A
|
| 578 |
+
# third of a step is 8.3 ms, which is the size of the
|
| 579 |
+
# constant late offset measured on chained clips. Whether
|
| 580 |
+
# it lands on or off the grid depends on the window
|
| 581 |
+
# length, the path, and the clip's own grid overhang, so
|
| 582 |
+
# it cycles rather than staying put. Rounding the end
|
| 583 |
+
# coordinate to the nearest integer costs at most a third
|
| 584 |
+
# of a step of placement and puts the pinned content on
|
| 585 |
+
# the same grid as the sound being generated from it.
|
| 586 |
+
end_coord = round(FRAME_RESCALE * end_frame)
|
| 587 |
+
end_frame = end_coord / FRAME_RESCALE
|
| 588 |
+
# Stock places a keyframe's audio window STARTING at
|
| 589 |
+
# FRAME_RESCALE * index past the target origin and running
|
| 590 |
+
# forward. We need it to END at the join, so the index is
|
| 591 |
+
# the start of a window `ref_audio_t` steps wide:
|
| 592 |
+
#
|
| 593 |
+
# start coord = FRAME_RESCALE * end_frame - ref_audio_t
|
| 594 |
+
# index = end_frame - ref_audio_t / FRAME_RESCALE
|
| 595 |
+
#
|
| 596 |
+
# which is fractional, and negative whenever the window is
|
| 597 |
+
# longer than the pinned head, which it normally is. Both
|
| 598 |
+
# are legal arithmetic in the layout and neither is
|
| 599 |
+
# reachable through the stock Add Guide node, so
|
| 600 |
+
# layout_contract checks them before the first render.
|
| 601 |
+
audio_kf = {
|
| 602 |
+
"resolved_frame_index": (end_frame
|
| 603 |
+
- ref_audio_t / FRAME_RESCALE),
|
| 604 |
+
"audio_latent": audio_latent,
|
| 605 |
+
}
|
| 606 |
+
audio_end_frame = end_frame
|
| 607 |
+
else:
|
| 608 |
+
# stock reference placement: the window sits in its own
|
| 609 |
+
# span ahead of the target, which is what makes the model
|
| 610 |
+
# imitate the sound rather than continue it. Kept as the
|
| 611 |
+
# comparison the timeline mode is measured against.
|
| 612 |
+
audio_ref = {
|
| 613 |
+
"kind": "audio",
|
| 614 |
+
"ref_audio_t": ref_audio_t,
|
| 615 |
+
"audio_latent": audio_latent,
|
| 616 |
+
}
|
| 617 |
+
|
| 618 |
+
# MERGE with any keyframes already on the conditioning instead of
|
| 619 |
+
# replacing them. A last_frame anchor from the upstream node, or an
|
| 620 |
+
# Add Guide anchor, is a legitimate companion to a chained head: the
|
| 621 |
+
# pinned run decides how the clip starts, the anchor decides where
|
| 622 |
+
# it ends. They need no special handling now that every keyframe
|
| 623 |
+
# carries its real index, ours included, and stock compensates all
|
| 624 |
+
# of them for references the same way.
|
| 625 |
+
#
|
| 626 |
+
# Anchors inside the pinned head are dropped: the pinned run
|
| 627 |
+
# already decides those frames, and a second cond block at the
|
| 628 |
+
# same coordinate would fight it.
|
| 629 |
+
head_end = span if anchor_mode == "head" else 0
|
| 630 |
+
tail_kfs = [audio_kf] if audio_kf is not None else []
|
| 631 |
+
out = []
|
| 632 |
+
dropped = []
|
| 633 |
+
for emb, extra in conditioning:
|
| 634 |
+
d = extra.copy()
|
| 635 |
+
prior = d.get("minimax_keyframes") or []
|
| 636 |
+
kept = []
|
| 637 |
+
for kf in prior:
|
| 638 |
+
p = kf.get("resolved_frame_index", 0)
|
| 639 |
+
if p >= frame_count:
|
| 640 |
+
raise ValueError(
|
| 641 |
+
"h3_motion_context: the conditioning carries a "
|
| 642 |
+
"keyframe anchored at frame %s, but this clip is "
|
| 643 |
+
"only %d frames. Wire the conditioning and the "
|
| 644 |
+
"latent from the same node." % (p, frame_count))
|
| 645 |
+
if p < head_end:
|
| 646 |
+
dropped.append(p)
|
| 647 |
+
continue
|
| 648 |
+
kept.append(dict(kf))
|
| 649 |
+
d["minimax_keyframes"] = kept + keyframes + tail_kfs
|
| 650 |
+
out.append([emb, d])
|
| 651 |
+
if dropped:
|
| 652 |
+
_LOG.warning(
|
| 653 |
+
"h3_motion_context: dropped %d keyframe anchor(s) at "
|
| 654 |
+
"frame(s) %s: the pinned head already decides frames "
|
| 655 |
+
"0..%d. A last_frame anchor is kept.",
|
| 656 |
+
len(dropped), sorted(set(dropped)), head_end - 1)
|
| 657 |
+
|
| 658 |
+
if audio_ref is not None:
|
| 659 |
+
out = node_helpers.conditioning_set_values(
|
| 660 |
+
out, {"minimax_refs": [audio_ref]}, append=True)
|
| 661 |
+
|
| 662 |
+
trim = span if anchor_mode == "head" else 0
|
| 663 |
+
_LOG.info("h3_motion_context: video from %s, %s/%s, %d frames -> %d "
|
| 664 |
+
"cond blocks at indices %d..%d, %d frame clip at %dx%d, "
|
| 665 |
+
"trim %d, audio %s",
|
| 666 |
+
video_src, encode_mode, anchor_mode, n, len(blocks),
|
| 667 |
+
indices[0], indices[-1], frame_count, width, height, trim,
|
| 668 |
+
("%d frames -> %d latent steps (%.3fs) from %s, %s"
|
| 669 |
+
% (a_frames, ref_audio_t, ref_audio_t / AUDIO_HZ, audio_src,
|
| 670 |
+
"on the timeline ending at frame %.3f" % audio_end_frame
|
| 671 |
+
if audio_end_frame is not None
|
| 672 |
+
else "stock ref placement"))
|
| 673 |
+
if ref_audio_t else "off")
|
| 674 |
+
return (out, trim)
|
| 675 |
+
|
| 676 |
+
|
| 677 |
+
class MiniMaxH3MotionContextTrim:
|
| 678 |
+
"""Drop the pinned head off a decoded clip, picture and sound together.
|
| 679 |
+
|
| 680 |
+
The pinned frames occupy the start of the delivered timeline, so they
|
| 681 |
+
have to come off before concatenating. Trimming only the images would
|
| 682 |
+
leave the audio a full trim_frames longer than the video, and muxing
|
| 683 |
+
those puts the whole soundtrack ahead of the picture by trim_frames/24
|
| 684 |
+
seconds. At 5 frames that is 208ms, silent on ambience but squarely
|
| 685 |
+
offbeat on anything with a pulse.
|
| 686 |
+
|
| 687 |
+
So this takes both streams and removes the same span from each: whole
|
| 688 |
+
frames from the images, the matching number of samples from the
|
| 689 |
+
waveform. Wire trim_frames from the motion context node so the count
|
| 690 |
+
follows whatever the encoder actually produced.
|
| 691 |
+
|
| 692 |
+
The tail needs the same treatment for a different reason. H3's audio
|
| 693 |
+
latent runs at 40 Hz against 24 fps picture, and FRAME_RESCALE is 5/3,
|
| 694 |
+
so the grid rarely lands on a frame boundary. It rounds to the
|
| 695 |
+
NEAREST step, which means a clip ships either about 8.3 ms more sound
|
| 696 |
+
than picture or about 8.3 ms less, depending on its length:
|
| 697 |
+
|
| 698 |
+
frames % 3 == 0 243 wants 405.00 steps, gets 405, exact
|
| 699 |
+
frames % 3 == 1 124 wants 206.67 steps, gets 207, sound is long
|
| 700 |
+
frames % 3 == 2 260 wants 433.33 steps, gets 433, sound is short
|
| 701 |
+
|
| 702 |
+
Either way the error compounds. Concatenate two clips and the second
|
| 703 |
+
seam is out by 16.7 ms, three and it is 25 ms, and it grows without
|
| 704 |
+
bound down a chain. It reads as a faint dampening at the first join
|
| 705 |
+
and a short click at later ones. Matching the tail to exactly
|
| 706 |
+
frames/fps stops it accumulating: a long tail is truncated, a short
|
| 707 |
+
one is zero-padded. The padded samples are sound the model never
|
| 708 |
+
generated, so silence is the only honest fill.
|
| 709 |
+
"""
|
| 710 |
+
|
| 711 |
+
@classmethod
|
| 712 |
+
def INPUT_TYPES(cls):
|
| 713 |
+
return {
|
| 714 |
+
"required": {
|
| 715 |
+
"images": ("IMAGE",),
|
| 716 |
+
"trim_frames": ("INT", {"default": 0, "min": 0, "max": 4096}),
|
| 717 |
+
},
|
| 718 |
+
"optional": {
|
| 719 |
+
"audio": ("AUDIO", {
|
| 720 |
+
"tooltip": "Decoded audio for the same clip. Trimmed by the "
|
| 721 |
+
"matching duration so sound stays locked to "
|
| 722 |
+
"picture. Leave unwired for silent clips."}),
|
| 723 |
+
"fps": ("FLOAT", {
|
| 724 |
+
"default": 24.0, "min": 1.0, "max": 240.0, "step": 0.001,
|
| 725 |
+
"tooltip": "Frame rate used to convert the trim into an "
|
| 726 |
+
"audio duration. Must match what you feed "
|
| 727 |
+
"Create Video."}),
|
| 728 |
+
"match_tail": ("BOOLEAN", {
|
| 729 |
+
"default": True,
|
| 730 |
+
"tooltip": "Make the audio duration equal frames/fps "
|
| 731 |
+
"exactly, trimming a long tail or padding a "
|
| 732 |
+
"short one with silence. H3 rounds its audio "
|
| 733 |
+
"grid to the nearest step, so each clip "
|
| 734 |
+
"carries about 8ms too much or too little "
|
| 735 |
+
"sound, which accumulates at every join in a "
|
| 736 |
+
"chain."}),
|
| 737 |
+
},
|
| 738 |
+
}
|
| 739 |
+
|
| 740 |
+
RETURN_TYPES = ("IMAGE", "AUDIO")
|
| 741 |
+
RETURN_NAMES = ("images", "audio")
|
| 742 |
+
FUNCTION = "trim"
|
| 743 |
+
CATEGORY = "conditioning/minimax"
|
| 744 |
+
DESCRIPTION = ("Remove the leading pinned frames from a decoded H3 clip, "
|
| 745 |
+
"trimming picture and sound by the same duration.")
|
| 746 |
+
|
| 747 |
+
def trim(self, images, trim_frames, audio=None, fps=24.0, match_tail=True):
|
| 748 |
+
n = max(0, int(trim_frames))
|
| 749 |
+
total = int(images.shape[0])
|
| 750 |
+
if n >= total:
|
| 751 |
+
raise ValueError(
|
| 752 |
+
"h3_motion_context: asked to trim %d frames from a %d frame clip"
|
| 753 |
+
% (n, total))
|
| 754 |
+
out_images = images[n:] if n else images
|
| 755 |
+
|
| 756 |
+
out_audio = audio
|
| 757 |
+
if audio is not None:
|
| 758 |
+
waveform = audio["waveform"]
|
| 759 |
+
sr = int(audio["sample_rate"])
|
| 760 |
+
seconds = n / float(fps)
|
| 761 |
+
cut = int(round(seconds * sr))
|
| 762 |
+
length = int(waveform.shape[-1])
|
| 763 |
+
if cut >= length:
|
| 764 |
+
raise ValueError(
|
| 765 |
+
"h3_motion_context: trimming %.3fs from %.3fs of audio would "
|
| 766 |
+
"leave nothing. Check that fps matches the clip."
|
| 767 |
+
% (seconds, length / sr))
|
| 768 |
+
waveform = waveform[..., cut:]
|
| 769 |
+
|
| 770 |
+
if match_tail:
|
| 771 |
+
frames_left = total - n
|
| 772 |
+
want = int(round(frames_left / float(fps) * sr))
|
| 773 |
+
have = int(waveform.shape[-1])
|
| 774 |
+
if have > want:
|
| 775 |
+
over = have - want
|
| 776 |
+
waveform = waveform[..., :want]
|
| 777 |
+
_LOG.info("h3_motion_context: tail trimmed %d samples "
|
| 778 |
+
"(%.2fms) so audio matches %d frames exactly",
|
| 779 |
+
over, over / sr * 1000.0, frames_left)
|
| 780 |
+
elif have < want:
|
| 781 |
+
# H3 rounds to the nearest audio step, so a third of
|
| 782 |
+
# clip lengths ship slightly LESS sound than picture
|
| 783 |
+
# rather than more. The missing samples are sound
|
| 784 |
+
# that was never generated, so zero is the honest
|
| 785 |
+
# fill; anything else would fabricate or attenuate
|
| 786 |
+
# real content to hide a seam. Leaving it short
|
| 787 |
+
# instead drifts every later clip earlier, and unlike
|
| 788 |
+
# the long case that error compounds down the chain.
|
| 789 |
+
# This also restores what the vae path assumes when
|
| 790 |
+
# it sets overhang to 0.
|
| 791 |
+
missing = want - have
|
| 792 |
+
waveform = torch.nn.functional.pad(waveform,
|
| 793 |
+
(0, missing))
|
| 794 |
+
_LOG.info("h3_motion_context: tail padded %d zero "
|
| 795 |
+
"samples (%.2fms) so audio matches %d "
|
| 796 |
+
"frames exactly",
|
| 797 |
+
missing, missing / sr * 1000.0, frames_left)
|
| 798 |
+
|
| 799 |
+
out_audio = {"waveform": waveform, "sample_rate": sr}
|
| 800 |
+
_LOG.info("h3_motion_context: %d frames / %.4fs picture, %.4fs sound, "
|
| 801 |
+
"drift %.2fms",
|
| 802 |
+
total - n, (total - n) / float(fps),
|
| 803 |
+
int(waveform.shape[-1]) / sr,
|
| 804 |
+
abs((total - n) / float(fps) - int(waveform.shape[-1]) / sr) * 1000.0)
|
| 805 |
+
elif n:
|
| 806 |
+
_LOG.info("h3_motion_context: trimmed %d leading frames, %d remain. "
|
| 807 |
+
"No audio wired; if this clip has sound, mux it through "
|
| 808 |
+
"this node or it will run %.3fs ahead of the picture.",
|
| 809 |
+
n, total - n, n / float(fps))
|
| 810 |
+
|
| 811 |
+
return (out_images, out_audio)
|
| 812 |
+
|
| 813 |
+
|
| 814 |
+
def _under_output(path):
|
| 815 |
+
"""Resolve `path` inside ComfyUI's output folder. None if it would escape.
|
| 816 |
+
|
| 817 |
+
Absolute paths are allowed only when they already live under output/.
|
| 818 |
+
Relative paths join onto output/. This is what keeps latent_path from
|
| 819 |
+
becoming an arbitrary file read or a delete-anywhere on Clear.
|
| 820 |
+
"""
|
| 821 |
+
root = os.path.realpath(folder_paths.get_output_directory())
|
| 822 |
+
p = (path or "").strip().strip('"').strip("'") or "h3_context"
|
| 823 |
+
resolved = os.path.realpath(p if os.path.isabs(p) else os.path.join(root, p))
|
| 824 |
+
if resolved != root and not resolved.startswith(root + os.sep):
|
| 825 |
+
return None
|
| 826 |
+
return resolved
|
| 827 |
+
|
| 828 |
+
|
| 829 |
+
def _resolve_latent_path(path, clip_index=0):
|
| 830 |
+
"""Turn the loader's path input into a concrete file under output/.
|
| 831 |
+
|
| 832 |
+
Accepts a path relative to ComfyUI's output folder, an absolute path
|
| 833 |
+
that already lives there, or a directory in either form. For a
|
| 834 |
+
directory, clip_index must be a positive slot: clip 1 is
|
| 835 |
+
*_00001.safetensors. Auto-mode files carry a trailing underscore
|
| 836 |
+
(*_00001_.safetensors) and are never matched, because their numbers
|
| 837 |
+
count runs and could hold a reject. clip_index 0 is handled by the
|
| 838 |
+
Load node itself (no file, first clip).
|
| 839 |
+
"""
|
| 840 |
+
c = _under_output(path)
|
| 841 |
+
if not c:
|
| 842 |
+
raise FileNotFoundError(
|
| 843 |
+
"h3_motion_context: path must stay inside the ComfyUI output folder.")
|
| 844 |
+
if os.path.isfile(c):
|
| 845 |
+
return c
|
| 846 |
+
if os.path.isdir(c):
|
| 847 |
+
idx = int(clip_index)
|
| 848 |
+
if idx <= 0:
|
| 849 |
+
raise FileNotFoundError(
|
| 850 |
+
"h3_motion_context: clip_index 0 does not load a file.")
|
| 851 |
+
# indexed slots use the natural name: clip 2 lives in
|
| 852 |
+
# *_00002.safetensors. Auto-mode files carry a trailing
|
| 853 |
+
# underscore (*_00002_.safetensors) and are deliberately
|
| 854 |
+
# NOT matched: their numbers count runs, not clips, so a
|
| 855 |
+
# reject could be sitting in any of them.
|
| 856 |
+
endings = ("_%05d.safetensors" % idx,
|
| 857 |
+
"_clip%03d.safetensors" % idx) # older versions
|
| 858 |
+
files = [os.path.join(c, f) for f in os.listdir(c)
|
| 859 |
+
if f.endswith(endings)]
|
| 860 |
+
if not files:
|
| 861 |
+
near = [f for f in os.listdir(c)
|
| 862 |
+
if f.endswith("_%05d_.safetensors" % idx)]
|
| 863 |
+
hint = ""
|
| 864 |
+
if near:
|
| 865 |
+
hint = (" Found %s, which is an auto-numbered save "
|
| 866 |
+
"(trailing underscore = numbered by RUN, so "
|
| 867 |
+
"it may be a reject). If it really is clip "
|
| 868 |
+
"%d, rename it to drop the trailing "
|
| 869 |
+
"underscore: %s" %
|
| 870 |
+
(near[0], idx,
|
| 871 |
+
near[0].replace("_%05d_" % idx,
|
| 872 |
+
"_%05d" % idx)))
|
| 873 |
+
raise FileNotFoundError(
|
| 874 |
+
"h3_motion_context: no saved latent for clip %d "
|
| 875 |
+
"(no *_%05d.safetensors in %s).%s"
|
| 876 |
+
% (idx, idx, c, hint))
|
| 877 |
+
return max(files, key=os.path.getmtime)
|
| 878 |
+
raise FileNotFoundError(
|
| 879 |
+
"h3_motion_context: %r is neither a file nor a folder under the "
|
| 880 |
+
"ComfyUI output directory." % ((path or "").strip() or "h3_context"))
|
| 881 |
+
|
| 882 |
+
|
| 883 |
+
def _latent_folder(path):
|
| 884 |
+
"""Directory that holds this chain's slot files, or None.
|
| 885 |
+
|
| 886 |
+
Refuses anything outside ComfyUI's output folder so Clear cannot
|
| 887 |
+
delete numbered safetensors elsewhere on disk.
|
| 888 |
+
"""
|
| 889 |
+
c = _under_output(path)
|
| 890 |
+
if not c:
|
| 891 |
+
return None
|
| 892 |
+
if os.path.isdir(c):
|
| 893 |
+
return c
|
| 894 |
+
if os.path.isfile(c):
|
| 895 |
+
return os.path.dirname(c)
|
| 896 |
+
return None
|
| 897 |
+
|
| 898 |
+
|
| 899 |
+
def _clip_slot_exists(latent_path, clip_index=1):
|
| 900 |
+
try:
|
| 901 |
+
_resolve_latent_path(latent_path, int(clip_index))
|
| 902 |
+
return True
|
| 903 |
+
except FileNotFoundError:
|
| 904 |
+
return False
|
| 905 |
+
|
| 906 |
+
|
| 907 |
+
_CHAIN_SLOT_FILE = re.compile(
|
| 908 |
+
r"(?:_\d{5}|_\d{5}_|_clip\d{3})\.safetensors(?:\.tmp)?$"
|
| 909 |
+
)
|
| 910 |
+
|
| 911 |
+
|
| 912 |
+
def _clear_clip_slots(latent_path):
|
| 913 |
+
"""Delete numbered chain slots. Custom filenames are left alone."""
|
| 914 |
+
folder = _latent_folder(latent_path)
|
| 915 |
+
if not folder:
|
| 916 |
+
return 0
|
| 917 |
+
removed = 0
|
| 918 |
+
for name in os.listdir(folder):
|
| 919 |
+
if not _CHAIN_SLOT_FILE.search(name):
|
| 920 |
+
continue
|
| 921 |
+
fp = os.path.join(folder, name)
|
| 922 |
+
try:
|
| 923 |
+
os.remove(fp)
|
| 924 |
+
except OSError:
|
| 925 |
+
gc.collect()
|
| 926 |
+
os.remove(fp)
|
| 927 |
+
removed += 1
|
| 928 |
+
if removed:
|
| 929 |
+
_LOG.info("h3_motion_context: cleared %d chain slot(s) from %s",
|
| 930 |
+
removed, folder)
|
| 931 |
+
return removed
|
| 932 |
+
|
| 933 |
+
|
| 934 |
+
def register_chain_routes():
|
| 935 |
+
"""POST slot_exists / clear_latents for the Chain node's start and Clear."""
|
| 936 |
+
try:
|
| 937 |
+
from aiohttp import web
|
| 938 |
+
from server import PromptServer
|
| 939 |
+
except ImportError:
|
| 940 |
+
return
|
| 941 |
+
server = getattr(PromptServer, "instance", None)
|
| 942 |
+
if server is None or getattr(register_chain_routes, "_done", False):
|
| 943 |
+
return
|
| 944 |
+
|
| 945 |
+
from .csrf_guard import require_same_origin
|
| 946 |
+
|
| 947 |
+
@server.routes.post("/h3_motion_context/slot_exists")
|
| 948 |
+
@require_same_origin
|
| 949 |
+
async def _slot_exists_route(request):
|
| 950 |
+
data = await request.json()
|
| 951 |
+
exists = _clip_slot_exists(data.get("latent_path") or "h3_context",
|
| 952 |
+
int(data.get("clip_index") or 1))
|
| 953 |
+
return web.json_response({"exists": exists})
|
| 954 |
+
|
| 955 |
+
@server.routes.post("/h3_motion_context/clear_latents")
|
| 956 |
+
@require_same_origin
|
| 957 |
+
async def _clear_latents_route(request):
|
| 958 |
+
data = await request.json()
|
| 959 |
+
n = _clear_clip_slots(data.get("latent_path") or "h3_context")
|
| 960 |
+
return web.json_response({"removed": n})
|
| 961 |
+
|
| 962 |
+
register_chain_routes._done = True
|
| 963 |
+
|
| 964 |
+
|
| 965 |
+
def _write_safetensors(path, tensors):
|
| 966 |
+
# safetensors load_file memory-maps on Windows. Overwriting a mapped
|
| 967 |
+
# slot (re-roll, or Load 0 aimed at the file Save is about to replace)
|
| 968 |
+
# fails with os error 1224. Write a sibling temp file and replace.
|
| 969 |
+
tmp = path + ".tmp"
|
| 970 |
+
try:
|
| 971 |
+
_st_save(tensors, tmp,
|
| 972 |
+
metadata={"format": "h3_motion_context_av_v1"})
|
| 973 |
+
try:
|
| 974 |
+
os.replace(tmp, path)
|
| 975 |
+
except OSError:
|
| 976 |
+
gc.collect()
|
| 977 |
+
os.replace(tmp, path)
|
| 978 |
+
finally:
|
| 979 |
+
try:
|
| 980 |
+
os.remove(tmp)
|
| 981 |
+
except OSError:
|
| 982 |
+
pass
|
| 983 |
+
|
| 984 |
+
|
| 985 |
+
class MiniMaxH3MotionContextSaveLatent:
|
| 986 |
+
"""Save an H3 AV latent to disk so the NEXT run can load it.
|
| 987 |
+
|
| 988 |
+
Wiring the sampler's output straight into context_latent is a cycle:
|
| 989 |
+
the sampler would be consuming its own result. The latent that motion
|
| 990 |
+
context needs is the PREVIOUS clip's, which lives in the previous run
|
| 991 |
+
-- so it has to cross runs through disk, the same way the frames and
|
| 992 |
+
audio already do. Stock Save/Load Latent can't serialise H3's nested
|
| 993 |
+
video/audio pair; this saves the two streams side by side.
|
| 994 |
+
"""
|
| 995 |
+
|
| 996 |
+
@classmethod
|
| 997 |
+
def INPUT_TYPES(cls):
|
| 998 |
+
return {
|
| 999 |
+
"required": {
|
| 1000 |
+
"latent": ("LATENT", {
|
| 1001 |
+
"tooltip": "The sampler's output latent (the same one "
|
| 1002 |
+
"you wire into the decode nodes)."}),
|
| 1003 |
+
"filename_prefix": ("STRING", {
|
| 1004 |
+
"default": "h3_context/clip",
|
| 1005 |
+
"tooltip": "Saved under the ComfyUI output folder. The "
|
| 1006 |
+
"default keeps all chain latents in one "
|
| 1007 |
+
"folder so the Load node can always pick "
|
| 1008 |
+
"the newest."}),
|
| 1009 |
+
"clip_index": ("INT", {
|
| 1010 |
+
"default": 1, "min": 0, "max": 9999, "step": 1,
|
| 1011 |
+
"tooltip": "Which clip of the chain THIS is. Saves to "
|
| 1012 |
+
"that clip's fixed slot, so a re-roll "
|
| 1013 |
+
"overwrites its own reject instead of "
|
| 1014 |
+
"stacking new files. First clip: 1 here "
|
| 1015 |
+
"and 0 on the Load node. Clip 2: 2 here "
|
| 1016 |
+
"and 1 on the Load node. 0 = old behaviour, "
|
| 1017 |
+
"a new numbered file every run (numbers "
|
| 1018 |
+
"count runs, not clips)."}),
|
| 1019 |
+
},
|
| 1020 |
+
}
|
| 1021 |
+
|
| 1022 |
+
RETURN_TYPES = ("STRING",)
|
| 1023 |
+
RETURN_NAMES = ("latent_path",)
|
| 1024 |
+
FUNCTION = "save"
|
| 1025 |
+
OUTPUT_NODE = True
|
| 1026 |
+
CATEGORY = "conditioning/minimax"
|
| 1027 |
+
DESCRIPTION = ("Save the sampler's AV latent so the next run's Motion "
|
| 1028 |
+
"Context node can pin audio from it via the matching "
|
| 1029 |
+
"Load node.")
|
| 1030 |
+
|
| 1031 |
+
def save(self, latent, filename_prefix, clip_index=0):
|
| 1032 |
+
if _st_save is None:
|
| 1033 |
+
raise RuntimeError("h3_motion_context: safetensors is not "
|
| 1034 |
+
"available; cannot save latents.")
|
| 1035 |
+
parts = _streams_from_latent(latent)
|
| 1036 |
+
if len(parts) < 2:
|
| 1037 |
+
raise ValueError(
|
| 1038 |
+
"h3_motion_context: latent has no audio stream; wire the "
|
| 1039 |
+
"sampler output of an H3 AV graph.")
|
| 1040 |
+
video = parts[0].cpu().contiguous()
|
| 1041 |
+
audio = parts[1].cpu().contiguous()
|
| 1042 |
+
folder, filename, counter, _, _ = folder_paths.get_save_image_path(
|
| 1043 |
+
filename_prefix, folder_paths.get_output_directory())
|
| 1044 |
+
if int(clip_index) > 0:
|
| 1045 |
+
# fixed slot with the natural name: clip 2 -> *_00002. A
|
| 1046 |
+
# re-roll of this clip overwrites its own save, so rejects
|
| 1047 |
+
# never accumulate or get loaded later. Auto mode (below)
|
| 1048 |
+
# keeps a trailing underscore, which is what excludes its
|
| 1049 |
+
# run-numbered files from indexed loading.
|
| 1050 |
+
path = os.path.join(folder, "%s_%05d.safetensors"
|
| 1051 |
+
% (filename, int(clip_index)))
|
| 1052 |
+
else:
|
| 1053 |
+
path = os.path.join(folder, "%s_%05d_.safetensors"
|
| 1054 |
+
% (filename, counter))
|
| 1055 |
+
_write_safetensors(path, {"video": video, "audio": audio})
|
| 1056 |
+
_LOG.info("h3_motion_context: saved AV latent to %s (video %s, "
|
| 1057 |
+
"audio %s)", path, tuple(video.shape), tuple(audio.shape))
|
| 1058 |
+
return (path,)
|
| 1059 |
+
|
| 1060 |
+
|
| 1061 |
+
class MiniMaxH3MotionContextLoadLatent:
|
| 1062 |
+
"""Load a saved H3 AV latent for the context_latent input.
|
| 1063 |
+
|
| 1064 |
+
clip_index means exactly what it says: set it to the clip you want to
|
| 1065 |
+
CONTINUE FROM, and that clip's slot is loaded. Generating clip 2 from
|
| 1066 |
+
clip 1: Load node 1, Save node 2. Use H3 Motion Context Chain to
|
| 1067 |
+
advance the pair and queue: Approve walks forward, Run/Re-roll stays
|
| 1068 |
+
on the current slot (use it instead of ComfyUI's Run), Chain is
|
| 1069 |
+
Approve on a loop (at 0/1 with no clip 1 yet it generates that
|
| 1070 |
+
first), Reset sets Load 0 / Save 1, Clear latents deletes numbered
|
| 1071 |
+
slots. All three nodes must sit in the same canvas group.
|
| 1072 |
+
|
| 1073 |
+
At 0 there is no previous clip: the loader returns nothing and Motion
|
| 1074 |
+
Context passes the conditioning through. First clip of a chain is
|
| 1075 |
+
Load 0 / Save 1, then Load 1 / Save 2, and so on.
|
| 1076 |
+
|
| 1077 |
+
The output is ONLY for the Motion Context node's context_latent input.
|
| 1078 |
+
It is not a decodable latent -- do not wire it into VAE decode.
|
| 1079 |
+
"""
|
| 1080 |
+
|
| 1081 |
+
@classmethod
|
| 1082 |
+
def INPUT_TYPES(cls):
|
| 1083 |
+
return {
|
| 1084 |
+
"required": {
|
| 1085 |
+
"latent_path": ("STRING", {
|
| 1086 |
+
"default": "h3_context",
|
| 1087 |
+
"tooltip": "A saved latent file, or a folder (relative "
|
| 1088 |
+
"paths resolve against the ComfyUI output "
|
| 1089 |
+
"directory). Pointing at a specific FILE "
|
| 1090 |
+
"always loads that file when clip_index "
|
| 1091 |
+
"is greater than 0, ignoring the index. "
|
| 1092 |
+
"clip_index 0 never reads a file."}),
|
| 1093 |
+
"clip_index": ("INT", {
|
| 1094 |
+
"default": 0, "min": 0, "max": 9999, "step": 1,
|
| 1095 |
+
"tooltip": "The clip to CONTINUE FROM: that clip's "
|
| 1096 |
+
"slot is loaded. First clip: 0 here and 1 "
|
| 1097 |
+
"on the Save node (nothing is loaded). "
|
| 1098 |
+
"Clip 2 from clip 1: 1 here and 2 on the "
|
| 1099 |
+
"Save node."}),
|
| 1100 |
+
},
|
| 1101 |
+
}
|
| 1102 |
+
|
| 1103 |
+
RETURN_TYPES = ("LATENT",)
|
| 1104 |
+
FUNCTION = "load"
|
| 1105 |
+
CATEGORY = "conditioning/minimax"
|
| 1106 |
+
DESCRIPTION = ("Load a latent saved by H3 Motion Context Save Latent, "
|
| 1107 |
+
"for the context_latent input only.")
|
| 1108 |
+
|
| 1109 |
+
@classmethod
|
| 1110 |
+
def IS_CHANGED(cls, latent_path, clip_index=0):
|
| 1111 |
+
# the path string stays constant while the file behind it changes
|
| 1112 |
+
# (an overwritten slot), so cache on the resolved file identity
|
| 1113 |
+
# instead -- otherwise ComfyUI would happily serve a stale latent
|
| 1114 |
+
# forever. Index 0 never reads a file.
|
| 1115 |
+
if int(clip_index) <= 0:
|
| 1116 |
+
return "disabled"
|
| 1117 |
+
try:
|
| 1118 |
+
p = _resolve_latent_path(latent_path, clip_index)
|
| 1119 |
+
return "%s:%d:owned" % (p, os.stat(p).st_mtime_ns)
|
| 1120 |
+
except Exception:
|
| 1121 |
+
return float("NaN") # unresolvable: never cache
|
| 1122 |
+
|
| 1123 |
+
def load(self, latent_path, clip_index=0):
|
| 1124 |
+
if int(clip_index) <= 0:
|
| 1125 |
+
return (None,)
|
| 1126 |
+
if _st_load is None:
|
| 1127 |
+
raise RuntimeError("h3_motion_context: safetensors is not "
|
| 1128 |
+
"available; cannot load latents.")
|
| 1129 |
+
path = _resolve_latent_path(latent_path, clip_index)
|
| 1130 |
+
data = _st_load(path)
|
| 1131 |
+
if "video" not in data or "audio" not in data:
|
| 1132 |
+
raise ValueError(
|
| 1133 |
+
"h3_motion_context: %s is not an h3_motion_context latent "
|
| 1134 |
+
"(missing video/audio streams). Was it saved by the stock "
|
| 1135 |
+
"Save Latent node instead?" % path)
|
| 1136 |
+
# copy off the mmap so a later Save can overwrite this slot on Windows
|
| 1137 |
+
video = data.pop("video").contiguous().clone()
|
| 1138 |
+
audio = data.pop("audio").contiguous().clone()
|
| 1139 |
+
_LOG.info("h3_motion_context: loaded AV latent from %s", path)
|
| 1140 |
+
# a plain list, not a NestedTensor: only this repo's context_latent
|
| 1141 |
+
# input accepts it, which is the point -- it cannot be mistaken
|
| 1142 |
+
# for a decodable latent without failing loudly downstream
|
| 1143 |
+
return ({"samples": [video, audio]},)
|
| 1144 |
+
|
| 1145 |
+
|
| 1146 |
+
class MiniMaxH3MotionContextChain:
|
| 1147 |
+
"""Approve, Run/Re-roll, auto-chain, reset indices, or clear slots.
|
| 1148 |
+
|
| 1149 |
+
Load, Save, and this node must sit in the same canvas group or the
|
| 1150 |
+
buttons do nothing. Chain is Approve on a loop. At Load 0 / Save 1
|
| 1151 |
+
with no clip 1 on disk it generates that first clip instead of
|
| 1152 |
+
advancing into a missing file. segments > 0 stops Chain after that
|
| 1153 |
+
many clips; 0 keeps going until Stop. Reset only sets 0/1. Clear
|
| 1154 |
+
latents deletes numbered chain slots, not custom filenames.
|
| 1155 |
+
Execute is a no-op; the buttons and two HTTP routes do the work.
|
| 1156 |
+
"""
|
| 1157 |
+
|
| 1158 |
+
@classmethod
|
| 1159 |
+
def INPUT_TYPES(cls):
|
| 1160 |
+
return {
|
| 1161 |
+
"required": {
|
| 1162 |
+
"segments": ("INT", {
|
| 1163 |
+
"default": 0, "min": 0, "max": 9999, "step": 1,
|
| 1164 |
+
"tooltip": "How many clips Chain generates before it "
|
| 1165 |
+
"stops. 0 keeps going until you click Stop."
|
| 1166 |
+
}),
|
| 1167 |
+
},
|
| 1168 |
+
}
|
| 1169 |
+
|
| 1170 |
+
RETURN_TYPES = ()
|
| 1171 |
+
FUNCTION = "noop"
|
| 1172 |
+
OUTPUT_NODE = True
|
| 1173 |
+
CATEGORY = "conditioning/minimax"
|
| 1174 |
+
DESCRIPTION = ("Approve advances Load/Save and runs the next clip. "
|
| 1175 |
+
"Run/Re-roll queues the current slot (use this instead "
|
| 1176 |
+
"of ComfyUI's Run). Chain is Approve on a loop; at "
|
| 1177 |
+
"Load 0 / Save 1 with no clip 1 saved it generates "
|
| 1178 |
+
"clip 1 first. segments is how many clips Chain runs "
|
| 1179 |
+
"before stopping (0 = until Stop). Reset sets Load 0 / "
|
| 1180 |
+
"Save 1. Clear latents deletes numbered chain slots. "
|
| 1181 |
+
"Load, Save, and this node must sit in the same canvas "
|
| 1182 |
+
"group or the buttons do nothing.")
|
| 1183 |
+
|
| 1184 |
+
def noop(self, segments=0):
|
| 1185 |
+
return ()
|
| 1186 |
+
|
| 1187 |
+
|
| 1188 |
+
NODE_CLASS_MAPPINGS = {
|
| 1189 |
+
"MiniMaxH3MotionContext": MiniMaxH3MotionContext,
|
| 1190 |
+
"MiniMaxH3MotionContextTrim": MiniMaxH3MotionContextTrim,
|
| 1191 |
+
"MiniMaxH3MotionContextSaveLatent": MiniMaxH3MotionContextSaveLatent,
|
| 1192 |
+
"MiniMaxH3MotionContextLoadLatent": MiniMaxH3MotionContextLoadLatent,
|
| 1193 |
+
"MiniMaxH3MotionContextChain": MiniMaxH3MotionContextChain,
|
| 1194 |
+
}
|
| 1195 |
+
NODE_DISPLAY_NAME_MAPPINGS = {
|
| 1196 |
+
"MiniMaxH3MotionContext": "H3 Motion Context",
|
| 1197 |
+
"MiniMaxH3MotionContextTrim": "H3 Motion Context Trim",
|
| 1198 |
+
"MiniMaxH3MotionContextSaveLatent": "H3 Motion Context Save Latent",
|
| 1199 |
+
"MiniMaxH3MotionContextLoadLatent": "H3 Motion Context Load Latent",
|
| 1200 |
+
"MiniMaxH3MotionContextChain": "H3 Motion Context Chain",
|
| 1201 |
+
}
|
custom_nodes/ComfyUI-H3-Motion-Context-main/probe_node.py
ADDED
|
@@ -0,0 +1,446 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Seam probe node: measure a join without leaving the graph.
|
| 2 |
+
|
| 3 |
+
seam_probe.py and level_step.py answer the two questions that matter at a
|
| 4 |
+
join, but both work on rendered files and both have to infer where the
|
| 5 |
+
seam is. That inference is the source of the constant ~8ms lag the CLI
|
| 6 |
+
probe reports: anchoring on a file's end rather than on the frame the
|
| 7 |
+
picture actually ends at is off by exactly the audio grid overhang, which
|
| 8 |
+
is a third of a step.
|
| 9 |
+
|
| 10 |
+
This node does not have to infer anything. It takes clip A as the same
|
| 11 |
+
latent you already pin from, and clip B before the trim, so the seam is
|
| 12 |
+
at a known sample and the pinned span is given rather than guessed. It
|
| 13 |
+
decodes A's audio and applies match_tail itself, which matters: comparing
|
| 14 |
+
against an untrimmed clip A would move the anchor by the overhang and
|
| 15 |
+
hand back a very convincing lag that is purely an artifact. Doing it
|
| 16 |
+
inside the node means that cannot be got wrong from the canvas.
|
| 17 |
+
|
| 18 |
+
Wire it inline: the audio VAE decode feeds this node, and this node feeds
|
| 19 |
+
the trim. The audio output is the input unchanged, so dropping it into an
|
| 20 |
+
existing chain changes nothing about the render.
|
| 21 |
+
|
| 22 |
+
clip_a_latent the same latent wired into Motion Context's
|
| 23 |
+
context_latent
|
| 24 |
+
audio_vae the same audio VAE
|
| 25 |
+
clip_b_untrimmed this clip's audio off the VAE decode, before trim
|
| 26 |
+
trim_frames from Motion Context, the same value the trim gets
|
| 27 |
+
|
| 28 |
+
Reported per join:
|
| 29 |
+
|
| 30 |
+
lag_ms how far B's reconstruction of the pinned span runs behind
|
| 31 |
+
A's original. Positive means B is late.
|
| 32 |
+
corr normalised cross-correlation over the pinned span. 1.0 is
|
| 33 |
+
identical shape, ~0 is unrelated. Low corr means the model
|
| 34 |
+
is imitating rather than continuing.
|
| 35 |
+
broadband RMS step across the cut, normalised to [0, 1].
|
| 36 |
+
floor the same for the 10th-percentile short-window RMS, which
|
| 37 |
+
tracks the bed under the content. This is the one that
|
| 38 |
+
catches a clip inventing its own silence.
|
| 39 |
+
|
| 40 |
+
Thresholds in the summary are the ones documented in the two scripts, not
|
| 41 |
+
new judgments. Numbers are always reported; the labels are a reading aid.
|
| 42 |
+
|
| 43 |
+
One limitation to know before trusting a lag figure. Cross-correlation
|
| 44 |
+
cannot tell a true alignment from one a whole cycle away, so on strongly
|
| 45 |
+
periodic content a lag larger than half a period reads as the nearest
|
| 46 |
+
alias instead: a 60 ms offset on material with an 18 ms period comes back
|
| 47 |
+
as -12.7 ms, confidently and at high correlation. Beat-driven music is
|
| 48 |
+
exactly this case. The tracker follows continuity between windows so a
|
| 49 |
+
drift cannot cycle-hop mid-span, and the first window takes the global
|
| 50 |
+
peak rather than assuming zero, which is what lets a uniform whole-step
|
| 51 |
+
offset be seen at all. But a large uniform offset on periodic content is
|
| 52 |
+
not something the method can resolve. Read a small lag as reliable and a
|
| 53 |
+
large one as "at least this much, possibly plus a whole number of
|
| 54 |
+
cycles".
|
| 55 |
+
"""
|
| 56 |
+
|
| 57 |
+
import logging
|
| 58 |
+
|
| 59 |
+
import numpy as np
|
| 60 |
+
|
| 61 |
+
from .nodes import FPS, _pixel_frames, _streams_from_latent
|
| 62 |
+
|
| 63 |
+
_LOG = logging.getLogger("h3_motion_context")
|
| 64 |
+
|
| 65 |
+
# from seam_probe.py: a match below this is not credible enough to track
|
| 66 |
+
CORR_CREDIBLE = 0.6
|
| 67 |
+
# from level_step.py
|
| 68 |
+
FLOOR_CLEAN, FLOOR_MARGINAL = 0.25, 0.50
|
| 69 |
+
BROADBAND_OK = 0.40
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def _mono(waveform):
|
| 73 |
+
"""Any [.., C, L] or [.., L, C] audio tensor -> mono float64 numpy.
|
| 74 |
+
|
| 75 |
+
The audio VAE's decode output orientation is not something to assume,
|
| 76 |
+
so the channel axis is identified by being the short one rather than
|
| 77 |
+
by position. Audio is thousands of samples long and at most a handful
|
| 78 |
+
of channels wide, so this is unambiguous in practice.
|
| 79 |
+
"""
|
| 80 |
+
a = waveform
|
| 81 |
+
if hasattr(a, "detach"):
|
| 82 |
+
a = a.detach()
|
| 83 |
+
if hasattr(a, "cpu"):
|
| 84 |
+
a = a.cpu()
|
| 85 |
+
a = np.asarray(a, dtype=np.float64)
|
| 86 |
+
while a.ndim > 2:
|
| 87 |
+
a = a[0]
|
| 88 |
+
if a.ndim == 2:
|
| 89 |
+
axis = 0 if a.shape[0] <= a.shape[1] else 1
|
| 90 |
+
a = a.mean(axis=axis)
|
| 91 |
+
return a
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def _norm_xcorr(win, ref):
|
| 95 |
+
"""Slide `win` across `ref`; return normalised correlation curve."""
|
| 96 |
+
win = win - win.mean()
|
| 97 |
+
ref = ref - ref.mean()
|
| 98 |
+
wn = np.sqrt((win ** 2).sum())
|
| 99 |
+
if wn < 1e-12:
|
| 100 |
+
return None
|
| 101 |
+
corr = np.correlate(ref, win, mode="valid")
|
| 102 |
+
csum = np.concatenate(([0.0], np.cumsum(ref ** 2)))
|
| 103 |
+
seg = csum[len(win):] - csum[:-len(win)]
|
| 104 |
+
denom = wn * np.sqrt(np.maximum(seg, 1e-12))
|
| 105 |
+
return corr / denom
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _tracked_peak(ncc, expect_idx):
|
| 109 |
+
"""Pick the alignment closest to the previous window's, among local
|
| 110 |
+
maxima within 10% of the best. Music is periodic, so the global argmax
|
| 111 |
+
can hop a whole cycle between windows and read as drift."""
|
| 112 |
+
best = float(ncc.max())
|
| 113 |
+
i = np.arange(1, len(ncc) - 1)
|
| 114 |
+
peaks = i[(ncc[i] >= ncc[i - 1]) & (ncc[i] >= ncc[i + 1]) &
|
| 115 |
+
(ncc[i] >= 0.9 * best)]
|
| 116 |
+
if len(peaks) == 0:
|
| 117 |
+
peaks = np.array([int(np.argmax(ncc))])
|
| 118 |
+
pick = int(peaks[np.argmin(np.abs(peaks - expect_idx))])
|
| 119 |
+
return pick, float(ncc[pick])
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def _rms(x):
|
| 123 |
+
if len(x) == 0:
|
| 124 |
+
return 0.0
|
| 125 |
+
return float(np.sqrt(np.mean(np.square(x))))
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def _floor_level(x, sr, win_ms=20.0, pct=10.0):
|
| 129 |
+
"""A low percentile of short-window RMS: the bed under the content."""
|
| 130 |
+
win = max(1, int(round(win_ms / 1000.0 * sr)))
|
| 131 |
+
n = len(x) // win
|
| 132 |
+
if n < 3:
|
| 133 |
+
return _rms(x)
|
| 134 |
+
trimmed = x[:n * win].reshape(n, win)
|
| 135 |
+
levels = np.sqrt(np.mean(np.square(trimmed), axis=1))
|
| 136 |
+
return float(np.percentile(levels, pct))
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def _step_ratio(a, b):
|
| 140 |
+
denom = a + b
|
| 141 |
+
if denom <= 1e-12:
|
| 142 |
+
return 0.0
|
| 143 |
+
return abs(a - b) / denom
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def _db(a, b):
|
| 147 |
+
if a <= 1e-12 or b <= 1e-12:
|
| 148 |
+
return float("inf")
|
| 149 |
+
return abs(20.0 * np.log10(a / b))
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
class MiniMaxH3MotionContextSeamProbe:
|
| 153 |
+
"""Measure a chain join: continuation quality and level step."""
|
| 154 |
+
|
| 155 |
+
@classmethod
|
| 156 |
+
def INPUT_TYPES(cls):
|
| 157 |
+
return {
|
| 158 |
+
"required": {
|
| 159 |
+
"clip_b_untrimmed": ("AUDIO", {
|
| 160 |
+
"tooltip": "This clip's audio straight off the VAE "
|
| 161 |
+
"decode, BEFORE the trim node. It still "
|
| 162 |
+
"carries the pinned head, which is what "
|
| 163 |
+
"gets compared."}),
|
| 164 |
+
"trim_frames": ("INT", {
|
| 165 |
+
"default": 0, "min": 0, "max": 4096,
|
| 166 |
+
"tooltip": "Wire this from the Motion Context node's "
|
| 167 |
+
"trim output, the same value the trim node "
|
| 168 |
+
"gets. It is the pinned span."}),
|
| 169 |
+
},
|
| 170 |
+
"optional": {
|
| 171 |
+
"clip_a_latent": ("LATENT", {
|
| 172 |
+
"tooltip": "The PREVIOUS clip's AV latent: the same "
|
| 173 |
+
"one wired into Motion Context's "
|
| 174 |
+
"context_latent. The node decodes and "
|
| 175 |
+
"tail-matches it itself. Without it only "
|
| 176 |
+
"clip B is described, nothing is "
|
| 177 |
+
"measured."}),
|
| 178 |
+
"audio_vae": ("VAE", {
|
| 179 |
+
"tooltip": "The H3 audio VAE, needed to decode clip "
|
| 180 |
+
"A's audio out of its latent."}),
|
| 181 |
+
"fps": ("FLOAT", {
|
| 182 |
+
"default": 24.0, "min": 1.0, "max": 240.0,
|
| 183 |
+
"step": 0.001,
|
| 184 |
+
"tooltip": "Must match what you feed Create Video."}),
|
| 185 |
+
"window_ms": ("FLOAT", {
|
| 186 |
+
"default": 50.0, "min": 5.0, "max": 500.0,
|
| 187 |
+
"step": 1.0,
|
| 188 |
+
"tooltip": "Correlation analysis window."}),
|
| 189 |
+
"search_ms": ("FLOAT", {
|
| 190 |
+
"default": 40.0, "min": 5.0, "max": 500.0,
|
| 191 |
+
"step": 1.0,
|
| 192 |
+
"tooltip": "Maximum lag searched either side."}),
|
| 193 |
+
},
|
| 194 |
+
}
|
| 195 |
+
|
| 196 |
+
RETURN_TYPES = ("AUDIO", "STRING")
|
| 197 |
+
RETURN_NAMES = ("audio", "report")
|
| 198 |
+
OUTPUT_TOOLTIPS = (
|
| 199 |
+
"clip_b_untrimmed, unchanged. Wire it on to the trim node.",
|
| 200 |
+
"The measurement report, for a Preview Text node.",
|
| 201 |
+
)
|
| 202 |
+
FUNCTION = "probe"
|
| 203 |
+
CATEGORY = "conditioning/minimax"
|
| 204 |
+
OUTPUT_NODE = True
|
| 205 |
+
DESCRIPTION = ("Measure a chain join in the graph: does clip B "
|
| 206 |
+
"continue clip A's audio, and does the level step at "
|
| 207 |
+
"the cut.")
|
| 208 |
+
|
| 209 |
+
def probe(self, clip_b_untrimmed, trim_frames, clip_a_latent=None,
|
| 210 |
+
audio_vae=None, fps=24.0, window_ms=50.0, search_ms=40.0):
|
| 211 |
+
passthrough = clip_b_untrimmed
|
| 212 |
+
sr = int(clip_b_untrimmed["sample_rate"])
|
| 213 |
+
b = _mono(clip_b_untrimmed["waveform"])
|
| 214 |
+
span_s = max(0, int(trim_frames)) / float(fps)
|
| 215 |
+
span = int(round(span_s * sr))
|
| 216 |
+
|
| 217 |
+
lines = ["H3 seam probe",
|
| 218 |
+
"clip B (untrimmed): %.4fs at %d Hz" % (len(b) / sr, sr),
|
| 219 |
+
"pinned span: %d frames = %.4fs = %d samples"
|
| 220 |
+
% (int(trim_frames), span_s, span)]
|
| 221 |
+
|
| 222 |
+
if span <= 0:
|
| 223 |
+
lines += ["", "trim_frames is 0, so there is no pinned span "
|
| 224 |
+
"and nothing to measure. Wire it from the "
|
| 225 |
+
"Motion Context node."]
|
| 226 |
+
return self._finish(passthrough, lines)
|
| 227 |
+
if len(b) < span:
|
| 228 |
+
lines += ["", "clip B is shorter than the pinned span. This "
|
| 229 |
+
"is the TRIMMED audio; wire the VAE decode "
|
| 230 |
+
"output instead."]
|
| 231 |
+
return self._finish(passthrough, lines)
|
| 232 |
+
if clip_a_latent is None or audio_vae is None:
|
| 233 |
+
missing = "clip_a_latent" if clip_a_latent is None else "audio_vae"
|
| 234 |
+
lines += ["", "no %s wired: measuring the join needs the "
|
| 235 |
+
"previous clip's latent and the audio VAE to "
|
| 236 |
+
"decode it." % missing]
|
| 237 |
+
return self._finish(passthrough, lines)
|
| 238 |
+
|
| 239 |
+
try:
|
| 240 |
+
a, raw, frames = self._clip_a_audio(clip_a_latent, audio_vae,
|
| 241 |
+
sr, fps)
|
| 242 |
+
except ValueError as exc:
|
| 243 |
+
lines += ["", str(exc)]
|
| 244 |
+
return self._finish(passthrough, lines)
|
| 245 |
+
|
| 246 |
+
pad = len(a) - len(raw)
|
| 247 |
+
lines.append("clip A: %d frames, decoded %.4fs, tail-matched to "
|
| 248 |
+
"%.4fs (%s %d samples)"
|
| 249 |
+
% (frames, len(raw) / sr, len(a) / sr,
|
| 250 |
+
"padded" if pad > 0 else "trimmed" if pad < 0
|
| 251 |
+
else "exact", abs(pad)))
|
| 252 |
+
lines.append("")
|
| 253 |
+
if len(a) < span:
|
| 254 |
+
lines.append("clip A is shorter than the pinned span, so "
|
| 255 |
+
"there is nothing to compare against.")
|
| 256 |
+
return self._finish(passthrough, lines)
|
| 257 |
+
lines += self._continuation(a, b, sr, span, window_ms, search_ms)
|
| 258 |
+
if pad > 0 and len(raw) >= span:
|
| 259 |
+
# Padding lengthened the array without moving the content, so
|
| 260 |
+
# the two anchors sit `pad` samples apart and one of them is
|
| 261 |
+
# wrong. Measuring both says which: a lag that vanishes
|
| 262 |
+
# against the raw end is an anchoring artifact, one that
|
| 263 |
+
# survives both is in the render.
|
| 264 |
+
alt = self._continuation(raw, b, sr, span, window_ms,
|
| 265 |
+
search_ms)
|
| 266 |
+
lines += ["", "cross-check: same audio anchored on the raw "
|
| 267 |
+
"decode instead of the padded end"]
|
| 268 |
+
lines += [" " + ln for ln in alt[1:]]
|
| 269 |
+
lines.append(" the padding is %.2f ms, so these two should "
|
| 270 |
+
"differ by that much. Whichever reads near zero "
|
| 271 |
+
"is the true alignment."
|
| 272 |
+
% (pad / sr * 1000.0,))
|
| 273 |
+
lines.append("")
|
| 274 |
+
lines += self._level(a, b, sr, span)
|
| 275 |
+
return self._finish(passthrough, lines)
|
| 276 |
+
|
| 277 |
+
@staticmethod
|
| 278 |
+
def _clip_a_audio(latent, audio_vae, sr, fps):
|
| 279 |
+
"""Decode clip A's audio and return (tail-matched, raw, frames).
|
| 280 |
+
|
| 281 |
+
The tail is the only part that gets read, and its position is
|
| 282 |
+
what the whole measurement is anchored on, so it has to sit
|
| 283 |
+
exactly where the delivered clip's does. H3 rounds its audio grid
|
| 284 |
+
to the nearest step, so a raw decode is up to a third of a step
|
| 285 |
+
long or short; leaving that in would shift every reported lag by
|
| 286 |
+
the same amount and look exactly like a real finding. The head
|
| 287 |
+
trim is not applied because it does not move the tail.
|
| 288 |
+
|
| 289 |
+
The raw decode comes back too, because tail-matching a SHORT clip
|
| 290 |
+
pads it, and padding moves the end of the array without moving
|
| 291 |
+
the content. On those clips the two arrays disagree about where
|
| 292 |
+
the anchor is by exactly the overhang, so the probe measures
|
| 293 |
+
against both and reports the difference rather than picking one
|
| 294 |
+
and hoping.
|
| 295 |
+
"""
|
| 296 |
+
parts = _streams_from_latent(latent)
|
| 297 |
+
if len(parts) < 2:
|
| 298 |
+
raise ValueError(
|
| 299 |
+
"clip_a_latent has no audio stream. Wire the sampler "
|
| 300 |
+
"output of an H3 AV graph, not a video-only latent.")
|
| 301 |
+
video, audio = parts[0], parts[1]
|
| 302 |
+
if video.ndim == 4:
|
| 303 |
+
video = video.unsqueeze(0)
|
| 304 |
+
frames = _pixel_frames(int(video.shape[2]))
|
| 305 |
+
if audio.ndim == 3:
|
| 306 |
+
audio = audio.unsqueeze(0)
|
| 307 |
+
decoded = audio_vae.decode(audio[:1])
|
| 308 |
+
a = _mono(decoded)
|
| 309 |
+
vae_sr = int(getattr(audio_vae, "audio_sample_rate", 32000))
|
| 310 |
+
if vae_sr != sr:
|
| 311 |
+
raise ValueError(
|
| 312 |
+
"clip A decodes at %d Hz but clip B is %d Hz. Not "
|
| 313 |
+
"comparable." % (vae_sr, sr))
|
| 314 |
+
want = int(round(frames / float(fps) * sr))
|
| 315 |
+
have = len(a)
|
| 316 |
+
raw = a
|
| 317 |
+
if have > want:
|
| 318 |
+
a = a[:want]
|
| 319 |
+
elif have < want:
|
| 320 |
+
a = np.concatenate([a, np.zeros(want - have)])
|
| 321 |
+
return a, raw, frames
|
| 322 |
+
|
| 323 |
+
def _continuation(self, a, b, sr, span, window_ms, search_ms):
|
| 324 |
+
"""B's first `span` samples should reconstruct A's last `span`."""
|
| 325 |
+
win = max(1, int(round(window_ms / 1000.0 * sr)))
|
| 326 |
+
hop = max(1, int(round(0.025 * sr))) # one audio latent step
|
| 327 |
+
search = max(1, int(round(search_ms / 1000.0 * sr)))
|
| 328 |
+
a_tail_start = len(a) - span
|
| 329 |
+
|
| 330 |
+
lags, corrs = [], []
|
| 331 |
+
t, prev_lag = 0, None
|
| 332 |
+
while t + win <= span:
|
| 333 |
+
centre = a_tail_start + t
|
| 334 |
+
lo = max(0, centre - search)
|
| 335 |
+
hi = min(len(a), centre + win + search)
|
| 336 |
+
ref = a[lo:hi]
|
| 337 |
+
if len(ref) <= win:
|
| 338 |
+
break
|
| 339 |
+
ncc = _norm_xcorr(b[t:t + win], ref)
|
| 340 |
+
if ncc is None:
|
| 341 |
+
t += hop
|
| 342 |
+
continue
|
| 343 |
+
# positive lag = B is LATE, matching an earlier part of A
|
| 344 |
+
if prev_lag is None:
|
| 345 |
+
# Nothing to track from yet, so take the global peak. Not
|
| 346 |
+
# zero: assuming the first window is aligned makes a join
|
| 347 |
+
# that is uniformly offset by a whole audio step read as
|
| 348 |
+
# the nearest cycle alias instead, which for periodic
|
| 349 |
+
# content can be a fraction of the real number. That is
|
| 350 |
+
# exactly the failure this probe exists to find.
|
| 351 |
+
pick = int(np.argmax(ncc))
|
| 352 |
+
corr = float(ncc[pick])
|
| 353 |
+
else:
|
| 354 |
+
expect = (centre - prev_lag) - lo
|
| 355 |
+
pick, corr = _tracked_peak(ncc, expect)
|
| 356 |
+
lag = centre - (lo + pick)
|
| 357 |
+
lags.append(lag / sr * 1000.0)
|
| 358 |
+
corrs.append(corr)
|
| 359 |
+
if corr > CORR_CREDIBLE:
|
| 360 |
+
prev_lag = lag
|
| 361 |
+
t += hop
|
| 362 |
+
|
| 363 |
+
out = ["continuation over the pinned span"]
|
| 364 |
+
if not corrs:
|
| 365 |
+
out.append(" window longer than the span; nothing analysed")
|
| 366 |
+
return out
|
| 367 |
+
corrs = np.array(corrs)
|
| 368 |
+
lags = np.array(lags)
|
| 369 |
+
strong = corrs > CORR_CREDIBLE
|
| 370 |
+
out.append(" mean corr %.3f (%d/%d windows above %.1f)"
|
| 371 |
+
% (corrs.mean(), int(strong.sum()), len(corrs),
|
| 372 |
+
CORR_CREDIBLE))
|
| 373 |
+
if strong.any():
|
| 374 |
+
sl = lags[strong]
|
| 375 |
+
out.append(" lag %+.2f ms mean, %+.2f to %+.2f "
|
| 376 |
+
"(credible windows only)"
|
| 377 |
+
% (sl.mean(), sl.min(), sl.max()))
|
| 378 |
+
at_edge = np.abs(sl) >= (search / sr * 1000.0 - 1.0)
|
| 379 |
+
if at_edge.any():
|
| 380 |
+
out.append(" %d window(s) pegged at the search limit; the "
|
| 381 |
+
"real lag is outside it. Raise search_ms."
|
| 382 |
+
% int(at_edge.sum()))
|
| 383 |
+
else:
|
| 384 |
+
out.append(" lag not reported: no window correlated "
|
| 385 |
+
"well enough to trust an alignment")
|
| 386 |
+
if corrs.mean() < 0.3:
|
| 387 |
+
out.append(" reading: the pinned audio is not being continued "
|
| 388 |
+
"at all. Check that context_latent or "
|
| 389 |
+
"context_audio is actually wired.")
|
| 390 |
+
elif corrs.mean() < CORR_CREDIBLE:
|
| 391 |
+
out.append(" reading: weak. The model is imitating the tail "
|
| 392 |
+
"rather than continuing it.")
|
| 393 |
+
elif strong.any() and abs(lags[strong].mean()) > 12.0:
|
| 394 |
+
out.append(" reading: locked but offset by more than half an "
|
| 395 |
+
"audio step. Placement, not continuation.")
|
| 396 |
+
else:
|
| 397 |
+
out.append(" reading: clean continuation.")
|
| 398 |
+
return out
|
| 399 |
+
|
| 400 |
+
def _level(self, a, b, sr, span):
|
| 401 |
+
"""The delivered join: A's tail against B's first post-trim audio."""
|
| 402 |
+
b_delivered = b[span:]
|
| 403 |
+
n = min(int(0.5 * sr), len(a), len(b_delivered))
|
| 404 |
+
out = ["level step across the delivered cut"]
|
| 405 |
+
if n < int(0.05 * sr):
|
| 406 |
+
out.append(" too little audio either side of the cut to "
|
| 407 |
+
"measure")
|
| 408 |
+
return out
|
| 409 |
+
left, right = a[-n:], b_delivered[:n]
|
| 410 |
+
bb = _step_ratio(_rms(left), _rms(right))
|
| 411 |
+
fl = _step_ratio(_floor_level(left, sr), _floor_level(right, sr))
|
| 412 |
+
out.append(" broadband %.3f (%.1f dB) %s"
|
| 413 |
+
% (bb, _db(_rms(left), _rms(right)),
|
| 414 |
+
"ok" if bb < BROADBAND_OK else "high"))
|
| 415 |
+
out.append(" floor %.3f (%.1f dB) %s"
|
| 416 |
+
% (fl, _db(_floor_level(left, sr),
|
| 417 |
+
_floor_level(right, sr)),
|
| 418 |
+
"clean" if fl < FLOOR_CLEAN
|
| 419 |
+
else "marginal" if fl < FLOOR_MARGINAL
|
| 420 |
+
else "audible step"))
|
| 421 |
+
out.append(" measured over %.0f ms either side" % (n / sr * 1000.0,))
|
| 422 |
+
if fl >= FLOOR_MARGINAL and bb < BROADBAND_OK:
|
| 423 |
+
out.append(" reading: the music matches but the bed underneath "
|
| 424 |
+
"restarts at the cut. Wire the previous clip's "
|
| 425 |
+
"audio into Motion Context.")
|
| 426 |
+
out.append(" note: the floor figure reads the quiet moments "
|
| 427 |
+
"between content, so wall to wall music with no rests "
|
| 428 |
+
"gives it nothing to measure and it collapses onto the "
|
| 429 |
+
"broadband number.")
|
| 430 |
+
return out
|
| 431 |
+
|
| 432 |
+
@staticmethod
|
| 433 |
+
def _finish(passthrough, lines):
|
| 434 |
+
report = "\n".join(lines)
|
| 435 |
+
for line in lines:
|
| 436 |
+
_LOG.info("h3_motion_context: %s", line)
|
| 437 |
+
return {"ui": {"text": [report]},
|
| 438 |
+
"result": (passthrough, report)}
|
| 439 |
+
|
| 440 |
+
|
| 441 |
+
NODE_CLASS_MAPPINGS = {
|
| 442 |
+
"MiniMaxH3MotionContextSeamProbe": MiniMaxH3MotionContextSeamProbe,
|
| 443 |
+
}
|
| 444 |
+
NODE_DISPLAY_NAME_MAPPINGS = {
|
| 445 |
+
"MiniMaxH3MotionContextSeamProbe": "H3 Motion Context Seam Probe",
|
| 446 |
+
}
|
custom_nodes/ComfyUI-H3-Motion-Context-main/pyproject.toml
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[project]
|
| 2 |
+
name = "comfyui-h3-motion-context"
|
| 3 |
+
description = "Clip chaining for MiniMax H3 in ComfyUI: pin the tail of the previous clip, picture and sound, so the next clip genuinely continues it. The picture is sliced out of the previous latent with no VAE round trip, and the pinned audio window ends at the join and reaches backwards, so the soundtrack continues rather than restarts. ComfyUI is not modified. Node ids MiniMaxH3MotionContext, MiniMaxH3MotionContextTrim, MiniMaxH3MotionContextSaveLatent, MiniMaxH3MotionContextLoadLatent, MiniMaxH3MotionContextChain, MiniMaxH3MotionContextSeamProbe."
|
| 4 |
+
version = "0.6.2"
|
| 5 |
+
license = { file = "LICENSE" }
|
| 6 |
+
readme = "README.md"
|
| 7 |
+
requires-python = ">=3.10"
|
| 8 |
+
authors = [
|
| 9 |
+
{ name = "NikoDemon80", email = "262659861+NikoDemon80@users.noreply.github.com" },
|
| 10 |
+
]
|
| 11 |
+
keywords = ["comfyui", "minimax-h3", "video-generation", "audio", "clip-chaining", "video-continuation"]
|
| 12 |
+
classifiers = [
|
| 13 |
+
"Development Status :: 4 - Beta",
|
| 14 |
+
"Intended Audience :: End Users/Desktop",
|
| 15 |
+
"License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
|
| 16 |
+
"Programming Language :: Python :: 3",
|
| 17 |
+
"Programming Language :: Python :: 3.10",
|
| 18 |
+
"Programming Language :: Python :: 3.11",
|
| 19 |
+
"Programming Language :: Python :: 3.12",
|
| 20 |
+
"Topic :: Multimedia :: Video",
|
| 21 |
+
"Topic :: Multimedia :: Sound/Audio",
|
| 22 |
+
]
|
| 23 |
+
|
| 24 |
+
[project.urls]
|
| 25 |
+
Homepage = "https://github.com/NikoDemon80/ComfyUI-H3-Motion-Context"
|
| 26 |
+
Repository = "https://github.com/NikoDemon80/ComfyUI-H3-Motion-Context"
|
| 27 |
+
Issues = "https://github.com/NikoDemon80/ComfyUI-H3-Motion-Context/issues"
|
| 28 |
+
|
| 29 |
+
[tool.comfy]
|
| 30 |
+
PublisherId = "nikodemon80"
|
| 31 |
+
DisplayName = "H3 Motion Context"
|
custom_nodes/ComfyUI-H3-Motion-Context-main/tests/_mock_harness.py
ADDED
|
@@ -0,0 +1,373 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Standalone harness for layout_contract, no ComfyUI and no GPU needed.
|
| 2 |
+
|
| 3 |
+
Fakes `comfy.ldm.minimax.model` with a PackedLayout that reproduces the
|
| 4 |
+
structure of the real one in comfy/ldm/minimax/model.py as of ComfyUI
|
| 5 |
+
0.33:
|
| 6 |
+
|
| 7 |
+
segment order text, then keyframe rows, then reference blocks, then
|
| 8 |
+
target audio, then target video. The target rows are
|
| 9 |
+
always the last two segments.
|
| 10 |
+
segment kinds text / cond / cond_audio / ref_img / ref_audio /
|
| 11 |
+
audio / video
|
| 12 |
+
position_ids [S, 3] float64, columns (t, h, w)
|
| 13 |
+
origin the target timeline starts past the span of every
|
| 14 |
+
reference block, and keyframe anchors are measured from
|
| 15 |
+
there, so stock compensates anchors for references
|
| 16 |
+
keyframe rows cond_t = origin + FRAME_RESCALE * resolved_frame_index.
|
| 17 |
+
A keyframe emits a cond segment only if it carries a
|
| 18 |
+
video latent, sized by that latent's step count, and a
|
| 19 |
+
cond_audio segment only if it carries an audio latent,
|
| 20 |
+
running FORWARD from cond_t for as many steps as the
|
| 21 |
+
latent has.
|
| 22 |
+
references laid out from a cursor starting at text_len:
|
| 23 |
+
image one ref_img segment, cursor += 1
|
| 24 |
+
audio one ref_audio segment of rt*2 rows,
|
| 25 |
+
channel-major stereo; cursor += rt, and
|
| 26 |
+
the segment is skipped when rt is 0
|
| 27 |
+
video, ref_audio then ref_img, both sharing the
|
| 28 |
+
video_audio cursor origin; cursor advances by
|
| 29 |
+
max(rt, sum of the video spans)
|
| 30 |
+
|
| 31 |
+
The pack no longer patches any of this. What it needs is that the
|
| 32 |
+
keyframe line above stays literal for a fractional, negative index, which
|
| 33 |
+
is how the pinned audio window is made to END at the join. No stock node
|
| 34 |
+
can produce such an index, so nothing upstream tests it. The mutation
|
| 35 |
+
knobs on make_mm are the ways that could plausibly stop being true, and
|
| 36 |
+
every one of them must be caught.
|
| 37 |
+
|
| 38 |
+
Checks:
|
| 39 |
+
1. the contract passes against a faithful 0.33 layout
|
| 40 |
+
2. an integer cast on the anchor index is caught
|
| 41 |
+
3. an audio window anchored at its end rather than its start is caught
|
| 42 |
+
4. a layout that stops compensating anchors for references is caught
|
| 43 |
+
5. a changed audio row count is caught
|
| 44 |
+
6. the older constructor is refused by signature, before anything is
|
| 45 |
+
built, with a message that describes the layout rather than naming a
|
| 46 |
+
ComfyUI version, and points at the pack version that runs on both
|
| 47 |
+
7. the failure is remembered, so a second render does not re-run the
|
| 48 |
+
checks and does not fail silently either
|
| 49 |
+
"""
|
| 50 |
+
|
| 51 |
+
import os
|
| 52 |
+
import sys
|
| 53 |
+
import types
|
| 54 |
+
|
| 55 |
+
import numpy as np
|
| 56 |
+
|
| 57 |
+
# the package dir has to be on sys.path so layout_contract imports by name
|
| 58 |
+
_PKG_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
| 59 |
+
if _PKG_DIR not in sys.path:
|
| 60 |
+
sys.path.insert(0, _PKG_DIR)
|
| 61 |
+
|
| 62 |
+
FRAME_RESCALE = 5.0 / 3.0
|
| 63 |
+
FRAME_PER_TOKEN = (1, 4, 4, 4, 4)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _video_t_spans(latent_t):
|
| 67 |
+
return [FRAME_RESCALE * FRAME_PER_TOKEN[k % 5] for k in range(latent_t)]
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def _frame_grid(h, w):
|
| 71 |
+
"""[n, 2] (h, w) coordinates for one latent frame's 2x2-patch rows.
|
| 72 |
+
|
| 73 |
+
The values only have to be deterministic and shaped right; nothing
|
| 74 |
+
under test reads these columns.
|
| 75 |
+
"""
|
| 76 |
+
n_h, n_w = h // 2, w // 2
|
| 77 |
+
hh, ww = np.meshgrid(np.arange(n_h, dtype=np.float64),
|
| 78 |
+
np.arange(n_w, dtype=np.float64), indexing="ij")
|
| 79 |
+
return np.stack([hh.reshape(-1), ww.reshape(-1)], axis=-1)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def _audio_grid(cursor, t, rows_per_step=2):
|
| 83 |
+
g = np.zeros((t * rows_per_step, 3), dtype=np.float64)
|
| 84 |
+
g[:, 0] = np.tile(cursor + np.arange(t, dtype=np.float64), rows_per_step)
|
| 85 |
+
if rows_per_step == 2:
|
| 86 |
+
g[:t, 2] = -1.0
|
| 87 |
+
g[t:, 2] = 1.0
|
| 88 |
+
return g
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _video_t_grid(n, origin):
|
| 92 |
+
spans = np.array(_video_t_spans(n), dtype=np.float64)
|
| 93 |
+
head = np.cumsum(spans[:-1]) if n > 1 else np.zeros(0, dtype=np.float64)
|
| 94 |
+
return float(origin) + np.concatenate(([0.0], head))
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def _video_grid(vt, frame, cursor):
|
| 98 |
+
g = np.empty((vt, frame.shape[0], 3), dtype=np.float64)
|
| 99 |
+
g[:, :, 0] = _video_t_grid(vt, cursor)[:, None]
|
| 100 |
+
g[:, :, 1:] = frame[None]
|
| 101 |
+
return g.reshape(-1, 3)
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def make_mm(audio_rows_per_step=2, int_index=False, audio_anchor="start",
|
| 105 |
+
compensate_refs=True, legacy=False):
|
| 106 |
+
"""Build a fake comfy.ldm.minimax.model.
|
| 107 |
+
|
| 108 |
+
Defaults reproduce ComfyUI 0.33. Each other argument breaks one
|
| 109 |
+
property the pack depends on:
|
| 110 |
+
|
| 111 |
+
int_index upstream adds a cast, and the fractional audio
|
| 112 |
+
index silently rounds
|
| 113 |
+
audio_anchor="end" the audio window is measured from its end rather
|
| 114 |
+
than its start, so it runs the wrong way
|
| 115 |
+
compensate_refs False puts the 0.32 behaviour back, where anchors
|
| 116 |
+
are computed from text_len and references slide
|
| 117 |
+
them relative to the target
|
| 118 |
+
audio_rows_per_step a change to how many rows one audio step takes
|
| 119 |
+
legacy the 0.32 constructor, frame_count and all
|
| 120 |
+
"""
|
| 121 |
+
mm = types.ModuleType("comfy.ldm.minimax.model")
|
| 122 |
+
mm.FRAME_RESCALE = FRAME_RESCALE
|
| 123 |
+
mm.FRAME_PER_TOKEN = FRAME_PER_TOKEN
|
| 124 |
+
mm._video_t_spans = _video_t_spans
|
| 125 |
+
|
| 126 |
+
def _ref_t_span(blk):
|
| 127 |
+
kind = blk["kind"]
|
| 128 |
+
if kind == "image":
|
| 129 |
+
return 1.0
|
| 130 |
+
if kind == "audio":
|
| 131 |
+
return float(blk["ref_audio_t"])
|
| 132 |
+
if kind in ("video", "video_audio"):
|
| 133 |
+
return max(float(blk["ref_audio_t"]),
|
| 134 |
+
sum(_video_t_spans(blk["latent_t"])))
|
| 135 |
+
raise ValueError("mock: unsupported ref kind %r" % kind)
|
| 136 |
+
|
| 137 |
+
def _build(self, text_len, latent_t, latent_h, latent_w, audio_t,
|
| 138 |
+
keyframes, refs, frame_count):
|
| 139 |
+
frame = _frame_grid(latent_h, latent_w)
|
| 140 |
+
frame_rows = frame.shape[0]
|
| 141 |
+
segs, blocks = [], []
|
| 142 |
+
|
| 143 |
+
def emit(kind, g):
|
| 144 |
+
segs.append((kind, g.shape[0]))
|
| 145 |
+
blocks.append(g)
|
| 146 |
+
|
| 147 |
+
g = np.zeros((text_len, 3), dtype=np.float64)
|
| 148 |
+
g[:, 0] = np.arange(text_len, dtype=np.float64)
|
| 149 |
+
emit("text", g)
|
| 150 |
+
|
| 151 |
+
origin = float(text_len)
|
| 152 |
+
if compensate_refs:
|
| 153 |
+
for blk in (refs or []):
|
| 154 |
+
origin += _ref_t_span(blk)
|
| 155 |
+
|
| 156 |
+
for kf in (keyframes or []):
|
| 157 |
+
p = kf["resolved_frame_index"]
|
| 158 |
+
if legacy:
|
| 159 |
+
if p == 0:
|
| 160 |
+
cond_t = float(text_len)
|
| 161 |
+
elif frame_count is not None and p == frame_count - 1:
|
| 162 |
+
cond_t = (float(text_len) + sum(_video_t_spans(latent_t))
|
| 163 |
+
- FRAME_RESCALE)
|
| 164 |
+
else:
|
| 165 |
+
raise ValueError(
|
| 166 |
+
"only first/last keyframe anchors are supported")
|
| 167 |
+
g = np.empty((frame_rows, 3), dtype=np.float64)
|
| 168 |
+
g[:, 0] = cond_t
|
| 169 |
+
g[:, 1:] = frame
|
| 170 |
+
emit("cond", g)
|
| 171 |
+
continue
|
| 172 |
+
if int_index:
|
| 173 |
+
p = int(p)
|
| 174 |
+
cond_t = origin + FRAME_RESCALE * p
|
| 175 |
+
video_latent = kf.get("latent")
|
| 176 |
+
if video_latent is not None:
|
| 177 |
+
emit("cond", _video_grid(video_latent.shape[2], frame, cond_t))
|
| 178 |
+
audio_latent = kf.get("audio_latent")
|
| 179 |
+
if audio_latent is not None:
|
| 180 |
+
rt = audio_latent.shape[-1]
|
| 181 |
+
start = cond_t if audio_anchor == "start" else cond_t - rt
|
| 182 |
+
emit("cond_audio", _audio_grid(start, rt, audio_rows_per_step))
|
| 183 |
+
|
| 184 |
+
cursor = float(text_len)
|
| 185 |
+
for blk in (refs or []):
|
| 186 |
+
kind = blk["kind"]
|
| 187 |
+
if kind == "image":
|
| 188 |
+
r_frame = _frame_grid(blk["latent_h"], blk["latent_w"])
|
| 189 |
+
g = np.empty((r_frame.shape[0], 3), dtype=np.float64)
|
| 190 |
+
g[:, 0] = cursor
|
| 191 |
+
g[:, 1:] = r_frame
|
| 192 |
+
emit("ref_img", g)
|
| 193 |
+
cursor += 1.0
|
| 194 |
+
elif kind == "audio":
|
| 195 |
+
rt = int(blk["ref_audio_t"])
|
| 196 |
+
if rt > 0:
|
| 197 |
+
emit("ref_audio",
|
| 198 |
+
_audio_grid(cursor, rt, audio_rows_per_step))
|
| 199 |
+
cursor += float(rt)
|
| 200 |
+
elif kind in ("video", "video_audio"):
|
| 201 |
+
rt = int(blk["ref_audio_t"])
|
| 202 |
+
vt = int(blk["latent_t"])
|
| 203 |
+
r_frame = _frame_grid(blk["latent_h"], blk["latent_w"])
|
| 204 |
+
if rt > 0:
|
| 205 |
+
emit("ref_audio",
|
| 206 |
+
_audio_grid(cursor, rt, audio_rows_per_step))
|
| 207 |
+
emit("ref_img", _video_grid(vt, r_frame, cursor))
|
| 208 |
+
cursor += max(float(rt), sum(_video_t_spans(vt)))
|
| 209 |
+
else:
|
| 210 |
+
raise ValueError("mock: unsupported ref kind %r" % kind)
|
| 211 |
+
|
| 212 |
+
# target audio then target video, always the last two segments
|
| 213 |
+
emit("audio", _audio_grid(cursor, audio_t))
|
| 214 |
+
emit("video", _video_grid(latent_t, frame, cursor))
|
| 215 |
+
|
| 216 |
+
seg_abs, off = [], 0
|
| 217 |
+
for kind, n in segs:
|
| 218 |
+
seg_abs.append((off, off + n, kind))
|
| 219 |
+
off += n
|
| 220 |
+
self.segments = seg_abs
|
| 221 |
+
self.seq_len = off
|
| 222 |
+
self.position_ids = np.concatenate(blocks)
|
| 223 |
+
|
| 224 |
+
# Two real signatures, not one with **kwargs: the pack decides whether
|
| 225 |
+
# this ComfyUI is too old by inspecting the signature, and 0.33 has to
|
| 226 |
+
# raise a real TypeError on frame_count.
|
| 227 |
+
if legacy:
|
| 228 |
+
class PackedLayout:
|
| 229 |
+
def __init__(self, text_len, latent_t, latent_h, latent_w,
|
| 230 |
+
audio_t, keyframes=None, refs=None,
|
| 231 |
+
frame_count=None):
|
| 232 |
+
_build(self, text_len, latent_t, latent_h, latent_w, audio_t,
|
| 233 |
+
keyframes, refs, frame_count)
|
| 234 |
+
else:
|
| 235 |
+
class PackedLayout:
|
| 236 |
+
def __init__(self, text_len, latent_t, latent_h, latent_w,
|
| 237 |
+
audio_t, keyframes=None, refs=None):
|
| 238 |
+
_build(self, text_len, latent_t, latent_h, latent_w, audio_t,
|
| 239 |
+
keyframes, refs, None)
|
| 240 |
+
|
| 241 |
+
mm.PackedLayout = PackedLayout
|
| 242 |
+
return mm
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
def make_torch():
|
| 246 |
+
t = types.ModuleType("torch")
|
| 247 |
+
t.equal = lambda a, b: a.shape == b.shape and bool(np.array_equal(a, b))
|
| 248 |
+
|
| 249 |
+
# nodes.py pads a short audio tail with torch.nn.functional.pad. Only
|
| 250 |
+
# the trailing-last-dim form is used, which is what this covers.
|
| 251 |
+
def _pad(x, pad, mode="constant", value=0.0):
|
| 252 |
+
before, after = int(pad[0]), int(pad[1])
|
| 253 |
+
widths = [(0, 0)] * (x.a.ndim - 1) + [(before, after)]
|
| 254 |
+
return type(x)(np.pad(x.a, widths, mode="constant",
|
| 255 |
+
constant_values=value))
|
| 256 |
+
|
| 257 |
+
functional = types.ModuleType("torch.nn.functional")
|
| 258 |
+
functional.pad = _pad
|
| 259 |
+
nn = types.ModuleType("torch.nn")
|
| 260 |
+
nn.functional = functional
|
| 261 |
+
t.nn = nn
|
| 262 |
+
return t
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
def load_contract(mm):
|
| 266 |
+
for name in ("comfy", "comfy.ldm", "comfy.ldm.minimax"):
|
| 267 |
+
sys.modules.setdefault(name, types.ModuleType(name))
|
| 268 |
+
sys.modules["comfy.ldm.minimax.model"] = mm
|
| 269 |
+
sys.modules["comfy"].ldm = sys.modules["comfy.ldm"]
|
| 270 |
+
sys.modules["comfy.ldm"].minimax = sys.modules["comfy.ldm.minimax"]
|
| 271 |
+
sys.modules["comfy.ldm.minimax"].model = mm
|
| 272 |
+
sys.modules["torch"] = make_torch()
|
| 273 |
+
sys.modules.pop("layout_contract", None)
|
| 274 |
+
import layout_contract
|
| 275 |
+
return layout_contract
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
def _refused(mm, label, expect):
|
| 279 |
+
"""The contract must refuse this layout, for the stated reason."""
|
| 280 |
+
lc = load_contract(mm)
|
| 281 |
+
try:
|
| 282 |
+
lc.ensure()
|
| 283 |
+
except RuntimeError as e:
|
| 284 |
+
assert expect in str(e), "%s: wrong reason: %s" % (label, e)
|
| 285 |
+
assert not lc.is_checked()
|
| 286 |
+
return str(e)
|
| 287 |
+
raise AssertionError("%s: the contract accepted it" % label)
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
def main():
|
| 291 |
+
# 1. faithful 0.33: the checks must pass and stay passed
|
| 292 |
+
mm = make_mm()
|
| 293 |
+
lc = load_contract(mm)
|
| 294 |
+
lc.ensure()
|
| 295 |
+
assert lc.is_checked()
|
| 296 |
+
lc.ensure() # second call is a no-op, not a second layout build
|
| 297 |
+
print("1. contract passes against a faithful ComfyUI 0.33 layout")
|
| 298 |
+
|
| 299 |
+
# 2. an int cast on the anchor index. The video anchors are whole
|
| 300 |
+
# numbers and survive it, so only the audio window moves, which is
|
| 301 |
+
# exactly the silent breakage this check exists for.
|
| 302 |
+
_refused(make_mm(int_index=True), "int cast",
|
| 303 |
+
"fractional or negative resolved_frame_index")
|
| 304 |
+
print("2. an integer cast on the anchor index is caught")
|
| 305 |
+
|
| 306 |
+
# 3. the window measured from its end instead of its start: it would
|
| 307 |
+
# run forwards from the join instead of backwards into it
|
| 308 |
+
_refused(make_mm(audio_anchor="end"), "audio anchored at end",
|
| 309 |
+
"pinned audio window")
|
| 310 |
+
print("3. an audio window anchored at its end is caught")
|
| 311 |
+
|
| 312 |
+
# 4. references stop compensating the anchors, as they did on 0.32
|
| 313 |
+
_refused(make_mm(compensate_refs=False), "no ref compensation",
|
| 314 |
+
"moved the anchors relative to the target")
|
| 315 |
+
print("4. a layout that stops compensating anchors for references is "
|
| 316 |
+
"caught")
|
| 317 |
+
|
| 318 |
+
# 5. a changed audio row count: the window is no longer the shape the
|
| 319 |
+
# placement assumes
|
| 320 |
+
_refused(make_mm(audio_rows_per_step=3), "audio rows",
|
| 321 |
+
"rows for")
|
| 322 |
+
print("5. a changed audio row count is caught")
|
| 323 |
+
|
| 324 |
+
# 6. the older constructor: refused by signature, before anything is
|
| 325 |
+
# built. The message must describe the LAYOUT, not guess a ComfyUI
|
| 326 |
+
# version. A wrapper's signature is not ComfyUI's, and the new layout
|
| 327 |
+
# sat on master unreleased for a while, so a version number inferred
|
| 328 |
+
# here would have been wrong in both directions.
|
| 329 |
+
msg = _refused(make_mm(legacy=True), "legacy", "older H3 layout")
|
| 330 |
+
assert "0.3.1" in msg, msg
|
| 331 |
+
assert "0.32" not in msg, "the refusal is guessing a ComfyUI version: %s" % msg
|
| 332 |
+
print("6. the older constructor refused by signature, pointing at pack "
|
| 333 |
+
"0.3.1 without guessing a ComfyUI version")
|
| 334 |
+
|
| 335 |
+
# 7. a failure is remembered. The node calls ensure() on every render,
|
| 336 |
+
# and the second call must raise the same way rather than quietly
|
| 337 |
+
# passing because some module state got left behind.
|
| 338 |
+
lc_bad = load_contract(make_mm(int_index=True))
|
| 339 |
+
for attempt in (1, 2):
|
| 340 |
+
try:
|
| 341 |
+
lc_bad.ensure()
|
| 342 |
+
except RuntimeError:
|
| 343 |
+
pass
|
| 344 |
+
else:
|
| 345 |
+
raise AssertionError("attempt %d did not raise" % attempt)
|
| 346 |
+
assert not lc_bad.is_checked()
|
| 347 |
+
print("7. a failed check stays failed on the next render")
|
| 348 |
+
|
| 349 |
+
# 8. the case that actually bit: another pack vendoring this project's
|
| 350 |
+
# old layout patch wraps the constructor, and that wrapper's signature
|
| 351 |
+
# carries frame_count on EVERY ComfyUI. Read naively it looks exactly
|
| 352 |
+
# like the older layout, and the user gets told to update a ComfyUI
|
| 353 |
+
# that is already current. Behaviour has to decide, not the signature.
|
| 354 |
+
mm = make_mm()
|
| 355 |
+
stock_init = mm.PackedLayout.__init__
|
| 356 |
+
|
| 357 |
+
def _patched_init(self, text_len, latent_t, latent_h, latent_w, audio_t,
|
| 358 |
+
keyframes=None, refs=None, frame_count=None):
|
| 359 |
+
stock_init(self, text_len, latent_t, latent_h, latent_w, audio_t,
|
| 360 |
+
keyframes=keyframes, refs=refs)
|
| 361 |
+
|
| 362 |
+
mm.PackedLayout.__init__ = _patched_init
|
| 363 |
+
lc = load_contract(mm)
|
| 364 |
+
lc.ensure()
|
| 365 |
+
assert lc.is_checked(), "a wrapped constructor was mistaken for the old layout"
|
| 366 |
+
print("8. a vendored copy of the old patch wrapping the constructor does "
|
| 367 |
+
"not get mistaken for an older ComfyUI")
|
| 368 |
+
|
| 369 |
+
print("all checks passed")
|
| 370 |
+
|
| 371 |
+
|
| 372 |
+
if __name__ == "__main__":
|
| 373 |
+
main()
|
custom_nodes/ComfyUI-H3-Motion-Context-main/tests/_node_smoke_test.py
ADDED
|
@@ -0,0 +1,676 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Smoke test: run MiniMaxH3MotionContext.apply() end to end with fakes.
|
| 2 |
+
|
| 3 |
+
Fakes ComfyUI's modules and tensor ops (numpy-backed) and drives the node
|
| 4 |
+
exactly as a graph would: a 124-frame clip at 480x864, 22 context frames,
|
| 5 |
+
audio from the previous clip's LATENT. Checks the produced conditioning
|
| 6 |
+
values: keyframe count and indices, the audio ref's step count, and the
|
| 7 |
+
fractional end_frame carrying the grid-overhang compensation.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import sys
|
| 11 |
+
import types
|
| 12 |
+
|
| 13 |
+
import numpy as np
|
| 14 |
+
|
| 15 |
+
import os
|
| 16 |
+
_TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 17 |
+
_PKG_DIR = os.path.dirname(_TESTS_DIR) # repo root, where the package lives
|
| 18 |
+
sys.path.insert(0, _TESTS_DIR)
|
| 19 |
+
from _mock_harness import make_mm, make_torch # noqa: E402
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class T:
|
| 23 |
+
"""Minimal numpy-backed tensor stand-in."""
|
| 24 |
+
|
| 25 |
+
def __init__(self, a):
|
| 26 |
+
self.a = np.asarray(a)
|
| 27 |
+
|
| 28 |
+
@property
|
| 29 |
+
def shape(self):
|
| 30 |
+
return self.a.shape
|
| 31 |
+
|
| 32 |
+
@property
|
| 33 |
+
def ndim(self):
|
| 34 |
+
return self.a.ndim
|
| 35 |
+
|
| 36 |
+
def __getitem__(self, idx):
|
| 37 |
+
return T(self.a[idx])
|
| 38 |
+
|
| 39 |
+
def movedim(self, src, dst):
|
| 40 |
+
return T(np.moveaxis(self.a, src, dst))
|
| 41 |
+
|
| 42 |
+
def unsqueeze(self, d):
|
| 43 |
+
return T(np.expand_dims(self.a, d))
|
| 44 |
+
|
| 45 |
+
def clone(self):
|
| 46 |
+
return T(self.a.copy())
|
| 47 |
+
|
| 48 |
+
def cpu(self):
|
| 49 |
+
return self
|
| 50 |
+
|
| 51 |
+
def contiguous(self):
|
| 52 |
+
return T(np.ascontiguousarray(self.a))
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class Nested:
|
| 56 |
+
def __init__(self, parts):
|
| 57 |
+
self.parts = parts
|
| 58 |
+
|
| 59 |
+
def unbind(self):
|
| 60 |
+
return list(self.parts)
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def main():
|
| 64 |
+
# fake modules the package imports
|
| 65 |
+
mm = make_mm()
|
| 66 |
+
for name in ("comfy", "comfy.ldm", "comfy.ldm.minimax"):
|
| 67 |
+
sys.modules.setdefault(name, types.ModuleType(name))
|
| 68 |
+
sys.modules["comfy.ldm.minimax.model"] = mm
|
| 69 |
+
sys.modules["comfy"].ldm = sys.modules["comfy.ldm"]
|
| 70 |
+
sys.modules["comfy.ldm"].minimax = sys.modules["comfy.ldm.minimax"]
|
| 71 |
+
sys.modules["comfy.ldm.minimax"].model = mm
|
| 72 |
+
sys.modules["torch"] = make_torch()
|
| 73 |
+
|
| 74 |
+
cu = types.ModuleType("comfy.utils")
|
| 75 |
+
cu.common_upscale = lambda s, w, h, m, c: T(
|
| 76 |
+
np.zeros((s.shape[0], 3, h, w), dtype=np.float32))
|
| 77 |
+
sys.modules["comfy.utils"] = cu
|
| 78 |
+
sys.modules["comfy"].utils = cu
|
| 79 |
+
|
| 80 |
+
mb = types.ModuleType("comfy.model_base")
|
| 81 |
+
|
| 82 |
+
class MiniMaxH3:
|
| 83 |
+
def extra_conds(self, **kw):
|
| 84 |
+
return {}
|
| 85 |
+
mb.MiniMaxH3 = MiniMaxH3
|
| 86 |
+
sys.modules["comfy.model_base"] = mb
|
| 87 |
+
sys.modules["comfy"].model_base = mb
|
| 88 |
+
|
| 89 |
+
captured = {}
|
| 90 |
+
nh = types.ModuleType("node_helpers")
|
| 91 |
+
|
| 92 |
+
def conditioning_set_values(cond, values, append=False):
|
| 93 |
+
# faithful to ComfyUI's node_helpers: copy each entry's dict, and
|
| 94 |
+
# when appending, concatenate onto whatever is already under the
|
| 95 |
+
# key instead of replacing it. The append path is what lets a
|
| 96 |
+
# Ref2VA graph keep its own reference blocks.
|
| 97 |
+
out = []
|
| 98 |
+
for t in cond:
|
| 99 |
+
n = [t[0], t[1].copy()]
|
| 100 |
+
for k, v in values.items():
|
| 101 |
+
if append:
|
| 102 |
+
old = n[1].get(k, None)
|
| 103 |
+
if old is not None:
|
| 104 |
+
v = old + v
|
| 105 |
+
n[1][k] = v
|
| 106 |
+
out.append(n)
|
| 107 |
+
captured.clear()
|
| 108 |
+
captured.update(out[0][1])
|
| 109 |
+
return out
|
| 110 |
+
nh.conditioning_set_values = conditioning_set_values
|
| 111 |
+
sys.modules["node_helpers"] = nh
|
| 112 |
+
|
| 113 |
+
import os
|
| 114 |
+
import tempfile
|
| 115 |
+
outdir = tempfile.mkdtemp()
|
| 116 |
+
fp = types.ModuleType("folder_paths")
|
| 117 |
+
fp.get_output_directory = lambda: outdir
|
| 118 |
+
|
| 119 |
+
def get_save_image_path(prefix, out, *a):
|
| 120 |
+
sub, name = os.path.split(prefix)
|
| 121 |
+
folder = os.path.join(out, sub)
|
| 122 |
+
os.makedirs(folder, exist_ok=True)
|
| 123 |
+
counter = 1 + sum(1 for f in os.listdir(folder)
|
| 124 |
+
if f.startswith(name))
|
| 125 |
+
return folder, name, counter, sub, prefix
|
| 126 |
+
fp.get_save_image_path = get_save_image_path
|
| 127 |
+
sys.modules["folder_paths"] = fp
|
| 128 |
+
|
| 129 |
+
st = types.ModuleType("safetensors")
|
| 130 |
+
stt = types.ModuleType("safetensors.torch")
|
| 131 |
+
|
| 132 |
+
def save_file(d, path, metadata=None):
|
| 133 |
+
np.savez(path + ".npz", **{k: v.a for k, v in d.items()})
|
| 134 |
+
open(path, "w").write(path + ".npz")
|
| 135 |
+
|
| 136 |
+
def load_file(path):
|
| 137 |
+
real = open(path).read()
|
| 138 |
+
z = np.load(real)
|
| 139 |
+
return {k: T(z[k]) for k in z.files}
|
| 140 |
+
stt.save_file, stt.load_file = save_file, load_file
|
| 141 |
+
st.torch = stt
|
| 142 |
+
sys.modules["safetensors"] = st
|
| 143 |
+
sys.modules["safetensors.torch"] = stt
|
| 144 |
+
|
| 145 |
+
# Load as a package so relative imports work whatever the folder is
|
| 146 |
+
# called (hyphens are not a valid module name).
|
| 147 |
+
pkg = types.ModuleType("h3mc_pkg")
|
| 148 |
+
pkg.__path__ = [_PKG_DIR]
|
| 149 |
+
pkg.__file__ = os.path.join(_PKG_DIR, "__init__.py")
|
| 150 |
+
sys.modules["h3mc_pkg"] = pkg
|
| 151 |
+
import h3mc_pkg.nodes as nodes
|
| 152 |
+
import h3mc_pkg.probe_node as probe_node
|
| 153 |
+
pkg.NODE_CLASS_MAPPINGS = dict(nodes.NODE_CLASS_MAPPINGS)
|
| 154 |
+
pkg.NODE_CLASS_MAPPINGS.update(probe_node.NODE_CLASS_MAPPINGS)
|
| 155 |
+
|
| 156 |
+
def audio_kf():
|
| 157 |
+
"""The one keyframe carrying pinned audio, from the last apply()."""
|
| 158 |
+
got = [k for k in captured["minimax_keyframes"]
|
| 159 |
+
if k.get("audio_latent") is not None]
|
| 160 |
+
assert len(got) == 1, "expected 1 pinned audio keyframe, got %d" % len(got)
|
| 161 |
+
return got[0]
|
| 162 |
+
|
| 163 |
+
def audio_end():
|
| 164 |
+
"""Target frame the pinned audio window ends at.
|
| 165 |
+
|
| 166 |
+
Stock places the window STARTING at the anchor index and running
|
| 167 |
+
forward, so the end is the index plus the window's own width in
|
| 168 |
+
pixel frames. That is the number the node solved for.
|
| 169 |
+
"""
|
| 170 |
+
kf = audio_kf()
|
| 171 |
+
return (kf["resolved_frame_index"]
|
| 172 |
+
+ kf["audio_latent"].shape[-1] / nodes.FRAME_RESCALE)
|
| 173 |
+
assert pkg.NODE_CLASS_MAPPINGS
|
| 174 |
+
# importing the pack must not have touched ComfyUI yet
|
| 175 |
+
assert not nodes._layout_checked(), \
|
| 176 |
+
"the layout checks ran at import time"
|
| 177 |
+
stock_init = mm.PackedLayout.__init__
|
| 178 |
+
|
| 179 |
+
# a 124-frame clip: latent_t 37 (7 full 17-frame groups + 1 + 4),
|
| 180 |
+
# audio grid ceil(124 * 5/3) = 207 steps, overhang exactly 1/3
|
| 181 |
+
latent_t, frames, audio_t = 37, 124, 207
|
| 182 |
+
assert nodes._pixel_frames(latent_t) == frames
|
| 183 |
+
h, w = 480 // 16, 864 // 16
|
| 184 |
+
target = {"samples": Nested([
|
| 185 |
+
T(np.zeros((1, 16, latent_t, h, w), dtype=np.float32)),
|
| 186 |
+
T(np.zeros((1, 32, 2, audio_t), dtype=np.float32)),
|
| 187 |
+
])}
|
| 188 |
+
# previous clip's sampler latent (same dims in this setup)
|
| 189 |
+
prev = {"samples": Nested([
|
| 190 |
+
T(np.arange(1 * 16 * latent_t * h * w, dtype=np.float32
|
| 191 |
+
).reshape(1, 16, latent_t, h, w)),
|
| 192 |
+
T(np.arange(1 * 32 * 2 * audio_t, dtype=np.float32
|
| 193 |
+
).reshape(1, 32, 2, audio_t)),
|
| 194 |
+
])}
|
| 195 |
+
context = T(np.zeros((124, 480, 864, 3), dtype=np.float32))
|
| 196 |
+
|
| 197 |
+
class VAE:
|
| 198 |
+
def encode(self, x):
|
| 199 |
+
n = x.shape[0]
|
| 200 |
+
steps = max(1, (n - 5) // 17 * 5 + 2)
|
| 201 |
+
return T(np.zeros((1, 16, steps, h, w), dtype=np.float32))
|
| 202 |
+
|
| 203 |
+
node = nodes.MiniMaxH3MotionContext()
|
| 204 |
+
|
| 205 |
+
def run(**kw):
|
| 206 |
+
"""apply(), capturing the conditioning it returns.
|
| 207 |
+
|
| 208 |
+
The node used to reach node_helpers to append its audio reference,
|
| 209 |
+
which is where these assertions used to read the result from. It
|
| 210 |
+
now builds the conditioning itself and touches no reference list,
|
| 211 |
+
so read what it actually hands the sampler.
|
| 212 |
+
"""
|
| 213 |
+
got = node.apply(**kw)
|
| 214 |
+
captured.clear()
|
| 215 |
+
captured.update(got[0][0][1])
|
| 216 |
+
return got
|
| 217 |
+
out, trim = run(
|
| 218 |
+
conditioning=[["c", {}]], vae=VAE(), latent=target,
|
| 219 |
+
context_frames=context, context_length="22",
|
| 220 |
+
audio_context_length=22, context_latent=prev)
|
| 221 |
+
|
| 222 |
+
# ... and running one must have checked the layout, and left it alone
|
| 223 |
+
assert nodes._layout_checked(), "the layout checks did not run"
|
| 224 |
+
assert mm.PackedLayout.__init__ is stock_init, \
|
| 225 |
+
"ComfyUI's constructor was modified"
|
| 226 |
+
|
| 227 |
+
kfs = captured["minimax_keyframes"]
|
| 228 |
+
# video keyframes carry a latent, the pinned audio is one more
|
| 229 |
+
# keyframe carrying only an audio latent, and it goes last
|
| 230 |
+
vid = [kf for kf in kfs if kf.get("latent") is not None]
|
| 231 |
+
aud = [kf for kf in kfs if kf.get("audio_latent") is not None]
|
| 232 |
+
assert len(kfs) == 8 and len(vid) == 7 and len(aud) == 1, len(kfs)
|
| 233 |
+
assert kfs[-1] is aud[0], "the audio keyframe is not last"
|
| 234 |
+
assert "minimax_refs" not in captured, \
|
| 235 |
+
"the pinned audio went out as a reference, not a keyframe"
|
| 236 |
+
idx = [kf["resolved_frame_index"] for kf in vid]
|
| 237 |
+
assert idx == [0, 1, 5, 9, 13, 17, 18], idx
|
| 238 |
+
assert "minimax_frame_count" not in captured
|
| 239 |
+
assert trim == 22
|
| 240 |
+
|
| 241 |
+
# the index is rt / FRAME_RESCALE frames before the end coordinate, so
|
| 242 |
+
# the window ENDS at the join rather than starting there. Here the
|
| 243 |
+
# audio and video windows are both 22 frames, so it starts exactly at
|
| 244 |
+
# frame 0; the 24-frame audio case below is the one that goes negative.
|
| 245 |
+
a_idx = aud[0]["resolved_frame_index"]
|
| 246 |
+
assert a_idx <= 0, a_idx
|
| 247 |
+
ref = {"ref_audio_t": aud[0]["audio_latent"].shape[-1],
|
| 248 |
+
"audio_latent": aud[0]["audio_latent"]}
|
| 249 |
+
assert ref["ref_audio_t"] == 37, ref["ref_audio_t"] # round(22/24*40)
|
| 250 |
+
tail = ref["audio_latent"]
|
| 251 |
+
assert tuple(tail.shape) == (1, 32, 2, 37), tail.shape
|
| 252 |
+
# tail must be the LAST 37 steps of the source
|
| 253 |
+
assert float(tail.a[0, 0, 0, -1]) == float(prev["samples"].parts[1]
|
| 254 |
+
.a[0, 0, 0, -1])
|
| 255 |
+
# the window must land on the target's own integer audio grid: the
|
| 256 |
+
# end coordinate is FRAME_RESCALE * end_frame and the target's audio
|
| 257 |
+
# rows sit on integers, so a fractional end coordinate would place
|
| 258 |
+
# the pinned content between them (a third of a step is 8.3 ms)
|
| 259 |
+
got_end = a_idx + ref["ref_audio_t"] / nodes.FRAME_RESCALE
|
| 260 |
+
end_coord = nodes.FRAME_RESCALE * got_end
|
| 261 |
+
assert abs(end_coord - round(end_coord)) < 1e-9, end_coord
|
| 262 |
+
assert abs(end_coord - 37.0) < 1e-9, end_coord
|
| 263 |
+
assert abs(got_end - 22.2) < 1e-6, got_end
|
| 264 |
+
# the pinned VIDEO must have come out of the latent too, not the VAE.
|
| 265 |
+
# The fake VAE returns zeros, so any nonzero block proves the source,
|
| 266 |
+
# and each block must equal the matching step of the source latent.
|
| 267 |
+
kf_blocks = [kf["latent"] for kf in vid]
|
| 268 |
+
src = prev["samples"].parts[0].a
|
| 269 |
+
start = latent_t - len(kf_blocks) # 37 - 7 = 30, cycle pos 0
|
| 270 |
+
assert start % 5 == 0, start
|
| 271 |
+
for k, blk in enumerate(kf_blocks):
|
| 272 |
+
assert tuple(blk.shape) == (1, 16, 1, h, w), blk.shape
|
| 273 |
+
assert np.array_equal(blk.a[0, :, 0], src[0, :, start + k]), k
|
| 274 |
+
assert nodes._steps_for_frames(22) == 7
|
| 275 |
+
assert nodes._steps_for_frames(5) == 2
|
| 276 |
+
assert nodes._steps_for_frames(39) == 12
|
| 277 |
+
assert nodes._steps_for_frames(1) == 1
|
| 278 |
+
assert nodes._steps_for_frames(3) is None # not a whole step boundary
|
| 279 |
+
print("latent path: 7 cond blocks at %s sliced from the latent tail "
|
| 280 |
+
"(bit-identical to the source steps), audio 37 steps, end_frame "
|
| 281 |
+
"%.4f (overhang-compensated)" % (idx, got_end))
|
| 282 |
+
|
| 283 |
+
# the recommended config pins 22 frames of picture and 24 of sound, so
|
| 284 |
+
# the audio window reaches back past the start of the clip. That index
|
| 285 |
+
# is negative AND fractional, which no stock node can produce and
|
| 286 |
+
# nothing upstream tests, so it is worth asserting here as well as in
|
| 287 |
+
# the layout contract.
|
| 288 |
+
run(conditioning=[["c", {}]], vae=VAE(), latent=target,
|
| 289 |
+
context_frames=context, context_length="22",
|
| 290 |
+
audio_context_length=24, context_latent=prev)
|
| 291 |
+
wide = audio_kf()
|
| 292 |
+
assert wide["audio_latent"].shape[-1] == 40, wide["audio_latent"].shape
|
| 293 |
+
w_idx = wide["resolved_frame_index"]
|
| 294 |
+
assert w_idx < 0 and w_idx != int(w_idx), w_idx
|
| 295 |
+
assert abs(w_idx - (37 - 40) / nodes.FRAME_RESCALE) < 1e-9, w_idx
|
| 296 |
+
w_end = nodes.FRAME_RESCALE * audio_end()
|
| 297 |
+
assert abs(w_end - round(w_end)) < 1e-9, w_end
|
| 298 |
+
print("wide audio window: 24 frames of sound against 22 of picture, "
|
| 299 |
+
"anchor index %.3f, still ending on the audio grid at %.1f"
|
| 300 |
+
% (w_idx, w_end))
|
| 301 |
+
|
| 302 |
+
# no latent wired: the pixel path, same offsets, and the fake VAE
|
| 303 |
+
# returns zeros so the blocks must now be zero rather than sliced.
|
| 304 |
+
# No audio either, so node_helpers is never called: read the
|
| 305 |
+
# conditioning the node returns rather than the capture hook
|
| 306 |
+
res, _ = run(
|
| 307 |
+
conditioning=[["c", {}]], vae=VAE(), latent=target,
|
| 308 |
+
context_frames=context, context_length="22",
|
| 309 |
+
audio_context_length=22)
|
| 310 |
+
px = res[0][1]["minimax_keyframes"]
|
| 311 |
+
assert [kf["resolved_frame_index"] for kf in px] == idx, \
|
| 312 |
+
"offsets differ by source"
|
| 313 |
+
assert float(px[0]["latent"].a.max()) == 0.0, "did not use the VAE"
|
| 314 |
+
print("pixels path: same %d blocks at the same offsets, encoded rather "
|
| 315 |
+
"than sliced" % len(px))
|
| 316 |
+
|
| 317 |
+
# a resolution change cannot slice a latent and must refuse, not
|
| 318 |
+
# quietly take the lossy path
|
| 319 |
+
small = {"samples": Nested([
|
| 320 |
+
T(np.zeros((1, 16, latent_t, h // 2, w // 2), dtype=np.float32)),
|
| 321 |
+
T(np.zeros((1, 32, 2, audio_t), dtype=np.float32)),
|
| 322 |
+
])}
|
| 323 |
+
try:
|
| 324 |
+
run(
|
| 325 |
+
conditioning=[["c", {}]], vae=VAE(), latent=target,
|
| 326 |
+
context_frames=context, context_length="22",
|
| 327 |
+
audio_context_length=22, context_latent=small)
|
| 328 |
+
except ValueError as e:
|
| 329 |
+
assert "cannot be resized" in str(e), str(e)
|
| 330 |
+
print("resolution change: refused, with the reason and the two "
|
| 331 |
+
"resolutions named")
|
| 332 |
+
else:
|
| 333 |
+
raise AssertionError("mismatched latent did not refuse")
|
| 334 |
+
|
| 335 |
+
# nothing wired: first clip of a chain. Pass the conditioning
|
| 336 |
+
# through and report trim_frames 0 so Trim is a no-op.
|
| 337 |
+
cond = [["c", {"untouched": True}]]
|
| 338 |
+
res, trim = run(
|
| 339 |
+
conditioning=cond, vae=VAE(), latent=target,
|
| 340 |
+
context_length="22", audio_context_length=22)
|
| 341 |
+
assert trim == 0, trim
|
| 342 |
+
assert res is cond
|
| 343 |
+
assert captured.get("untouched") is True
|
| 344 |
+
assert "minimax_keyframes" not in captured
|
| 345 |
+
print("nothing wired: original conditioning passed through, "
|
| 346 |
+
"trim_frames 0")
|
| 347 |
+
|
| 348 |
+
# the constants that replaced the widgets must be on the good values
|
| 349 |
+
assert nodes.ENCODE_MODE == "video"
|
| 350 |
+
assert nodes.ANCHOR_MODE == "head"
|
| 351 |
+
assert nodes.AUDIO_MODE == "timeline"
|
| 352 |
+
assert nodes.CROP == "disabled"
|
| 353 |
+
|
| 354 |
+
# the cycle-position property the whole latent video path rests on,
|
| 355 |
+
# checked across every clip length and window rather than argued for
|
| 356 |
+
for g in range(1, 40):
|
| 357 |
+
steps_total = 5 * g + 2
|
| 358 |
+
frames_total = nodes._pixel_frames(steps_total)
|
| 359 |
+
assert (frames_total - 5) % 17 == 0, (g, frames_total)
|
| 360 |
+
for wnd in (5, 22, 39, 56):
|
| 361 |
+
st = nodes._steps_for_frames(wnd)
|
| 362 |
+
if st is None or st > steps_total:
|
| 363 |
+
continue
|
| 364 |
+
begin = steps_total - st
|
| 365 |
+
assert begin % 5 == 0, (frames_total, wnd, begin % 5)
|
| 366 |
+
assert nodes._pixel_frames(st) == wnd
|
| 367 |
+
# offsets computed for a fresh run must match the sliced run
|
| 368 |
+
assert (nodes._step_offsets(st)
|
| 369 |
+
== [nodes._pixel_frames(k) for k in range(st)])
|
| 370 |
+
print("cycle check: every clip length 22..%d x windows 5/22/39/56 "
|
| 371 |
+
"slices from cycle position 0" % nodes._pixel_frames(5 * 39 + 2))
|
| 372 |
+
|
| 373 |
+
# decoded-audio path must still work and carry integer end_frame
|
| 374 |
+
captured.clear()
|
| 375 |
+
|
| 376 |
+
class AudioVAE:
|
| 377 |
+
audio_sample_rate = 32000
|
| 378 |
+
|
| 379 |
+
def encode(self, x):
|
| 380 |
+
steps = int(round(x.shape[-2] / 32000 * 40))
|
| 381 |
+
return T(np.zeros((1, 32, 2, steps), dtype=np.float32))
|
| 382 |
+
|
| 383 |
+
audio = {"waveform": T(np.zeros((1, 2, 32000), dtype=np.float32)),
|
| 384 |
+
"sample_rate": 32000}
|
| 385 |
+
run(
|
| 386 |
+
conditioning=[["c", {}]], vae=VAE(), latent=target,
|
| 387 |
+
context_frames=context, context_length="22",
|
| 388 |
+
audio_context_length=22, audio_vae=AudioVAE(), context_audio=audio)
|
| 389 |
+
# no overhang to compensate on this path, but the same grid rule
|
| 390 |
+
# applies: 22 frames is 36.667 steps, which must snap to 37
|
| 391 |
+
end2 = nodes.FRAME_RESCALE * audio_end()
|
| 392 |
+
assert abs(end2 - 37.0) < 1e-9, end2
|
| 393 |
+
print("vae path: window snapped to the audio grid, end coord %.1f" % end2)
|
| 394 |
+
|
| 395 |
+
# every clip length on the ladder, every window, both paths: the
|
| 396 |
+
# pinned window must always end on an integer audio coordinate
|
| 397 |
+
for g in range(2, 13):
|
| 398 |
+
f = 17 * g + 5
|
| 399 |
+
# nearest, not ceil: ceil would simulate +2/3 on the lengths that
|
| 400 |
+
# actually run -1/3, so this loop used to exercise a value the
|
| 401 |
+
# model never produces
|
| 402 |
+
at = round(nodes.FRAME_RESCALE * f)
|
| 403 |
+
oh = at - nodes.FRAME_RESCALE * f
|
| 404 |
+
for span in (5, 22, 39, 56):
|
| 405 |
+
for ohv in (0.0, oh):
|
| 406 |
+
ef = span + ohv / nodes.FRAME_RESCALE
|
| 407 |
+
ec = round(nodes.FRAME_RESCALE * ef)
|
| 408 |
+
ef = ec / nodes.FRAME_RESCALE
|
| 409 |
+
c = nodes.FRAME_RESCALE * ef
|
| 410 |
+
assert abs(c - round(c)) < 1e-9, (f, span, ohv, c)
|
| 411 |
+
print("grid check: every ladder length x window 5/22/39/56 x both "
|
| 412 |
+
"audio paths ends on an integer audio coordinate")
|
| 413 |
+
|
| 414 |
+
# All three frame-count residues, latent path, 22-frame window. The
|
| 415 |
+
# 124-frame case above is frames % 3 == 1, the one residue where
|
| 416 |
+
# rounding up and rounding to nearest agree, so on its own it cannot
|
| 417 |
+
# tell the two rules apart. 260 frames is the negative-overhang case:
|
| 418 |
+
# compensation moves the end coordinate to 36.33, which rounds to 36,
|
| 419 |
+
# where failing open would leave it at 36.67 and round to 37, a whole
|
| 420 |
+
# audio step (25 ms) late.
|
| 421 |
+
import logging as _logging
|
| 422 |
+
|
| 423 |
+
class _Catcher(_logging.Handler):
|
| 424 |
+
def __init__(self):
|
| 425 |
+
_logging.Handler.__init__(self)
|
| 426 |
+
self.msgs = []
|
| 427 |
+
|
| 428 |
+
def emit(self, record):
|
| 429 |
+
self.msgs.append(record.getMessage())
|
| 430 |
+
|
| 431 |
+
catcher = _Catcher()
|
| 432 |
+
catcher.setLevel(_logging.WARNING)
|
| 433 |
+
nodes._LOG.addHandler(catcher)
|
| 434 |
+
try:
|
| 435 |
+
for lt, f, at, want_coord, want_end in (
|
| 436 |
+
(37, 124, 207, 37, 22.2), # +1/3, reaches past
|
| 437 |
+
(72, 243, 405, 37, 22.2), # 0, exact
|
| 438 |
+
(77, 260, 433, 36, 21.6)): # -1/3, falls short
|
| 439 |
+
assert nodes._pixel_frames(lt) == f, (lt, f)
|
| 440 |
+
assert at == round(nodes.FRAME_RESCALE * f), (f, at)
|
| 441 |
+
captured.clear()
|
| 442 |
+
del catcher.msgs[:]
|
| 443 |
+
tgt = {"samples": Nested([
|
| 444 |
+
T(np.zeros((1, 16, lt, h, w), dtype=np.float32)),
|
| 445 |
+
T(np.zeros((1, 32, 2, at), dtype=np.float32)),
|
| 446 |
+
])}
|
| 447 |
+
src = {"samples": Nested([
|
| 448 |
+
T(np.arange(1 * 16 * lt * h * w, dtype=np.float32
|
| 449 |
+
).reshape(1, 16, lt, h, w)),
|
| 450 |
+
T(np.arange(1 * 32 * 2 * at, dtype=np.float32
|
| 451 |
+
).reshape(1, 32, 2, at)),
|
| 452 |
+
])}
|
| 453 |
+
run(
|
| 454 |
+
conditioning=[["c", {}]], vae=VAE(), latent=tgt,
|
| 455 |
+
context_frames=T(np.zeros((f, 480, 864, 3),
|
| 456 |
+
dtype=np.float32)),
|
| 457 |
+
context_length="22", audio_context_length=22,
|
| 458 |
+
context_latent=src)
|
| 459 |
+
got = audio_end()
|
| 460 |
+
coord = nodes.FRAME_RESCALE * got
|
| 461 |
+
assert abs(coord - round(coord)) < 1e-9, (f, coord)
|
| 462 |
+
assert abs(coord - want_coord) < 1e-9, (f, coord, want_coord)
|
| 463 |
+
assert abs(got - want_end) < 1e-6, (f, got, want_end)
|
| 464 |
+
# the grid is legal on all three, so nothing may warn about it
|
| 465 |
+
assert not [m for m in catcher.msgs if "audio grid" in m], \
|
| 466 |
+
(f, catcher.msgs)
|
| 467 |
+
finally:
|
| 468 |
+
nodes._LOG.removeHandler(catcher)
|
| 469 |
+
print("residue check: frames % 3 in 0/1/2 all accepted, end coords "
|
| 470 |
+
"37/37/36 (the -1/3 case would be 37 if it failed open)")
|
| 471 |
+
|
| 472 |
+
# The trim node's match_tail: a long tail is truncated, a short one is
|
| 473 |
+
# padded, both to exactly frames/fps. The short case only occurs on
|
| 474 |
+
# frames % 3 == 2 clips, which is why it went unexercised.
|
| 475 |
+
trimmer = nodes.MiniMaxH3MotionContextTrim()
|
| 476 |
+
for f, direction in ((124, "long"), (243, "exact"), (260, "short")):
|
| 477 |
+
sr = 32000
|
| 478 |
+
# what H3 actually ships: the audio grid at nearest rounding,
|
| 479 |
+
# decoded back to samples
|
| 480 |
+
steps = round(nodes.FRAME_RESCALE * f)
|
| 481 |
+
have = int(round(steps / nodes.AUDIO_HZ * sr))
|
| 482 |
+
want = int(round((f - 22) / 24.0 * sr))
|
| 483 |
+
imgs = T(np.zeros((f, 8, 8, 3), dtype=np.float32))
|
| 484 |
+
wav = T(np.zeros((1, 2, have), dtype=np.float32))
|
| 485 |
+
_, out = trimmer.trim(
|
| 486 |
+
images=imgs, trim_frames=22,
|
| 487 |
+
audio={"waveform": wav, "sample_rate": sr}, fps=24.0,
|
| 488 |
+
match_tail=True)
|
| 489 |
+
got = int(out["waveform"].shape[-1])
|
| 490 |
+
assert got == want, (f, direction, got, want)
|
| 491 |
+
# and the drift the node reports must be under half a sample
|
| 492 |
+
assert abs(got / sr - (f - 22) / 24.0) * 1000.0 < 0.02, (f, got)
|
| 493 |
+
print("match_tail check: long tail trimmed, short tail padded, all "
|
| 494 |
+
"three residues land on the exact sample count")
|
| 495 |
+
|
| 496 |
+
# Ref2VA: a graph whose conditioning already carries reference blocks
|
| 497 |
+
# must keep every one of them, with the motion context audio block
|
| 498 |
+
# appended rather than replacing the lot. Before this, the node
|
| 499 |
+
# assigned minimax_refs outright and an R2V graph silently lost its
|
| 500 |
+
# references at the moment chaining was switched on.
|
| 501 |
+
captured.clear()
|
| 502 |
+
existing = [
|
| 503 |
+
{"kind": "image", "latent_h": 8, "latent_w": 12},
|
| 504 |
+
{"kind": "video_audio", "latent_h": 8, "latent_w": 12,
|
| 505 |
+
"latent_t": 3, "ref_audio_t": 5},
|
| 506 |
+
{"kind": "audio", "ref_audio_t": 9},
|
| 507 |
+
]
|
| 508 |
+
r2v_cond = [["c", {"minimax_refs": [dict(r) for r in existing]}]]
|
| 509 |
+
out_cond, _ = run(
|
| 510 |
+
conditioning=r2v_cond, vae=VAE(), latent=target,
|
| 511 |
+
context_frames=context, context_length="22",
|
| 512 |
+
audio_context_length=22, context_latent=prev)
|
| 513 |
+
refs_out = captured["minimax_refs"]
|
| 514 |
+
assert len(refs_out) == 3, len(refs_out)
|
| 515 |
+
for got, want in zip(refs_out, existing):
|
| 516 |
+
assert got["kind"] == want["kind"], (got, want)
|
| 517 |
+
# the pinned audio is a keyframe now, so the graph's reference list is
|
| 518 |
+
# not touched at all. Its blocks still push the target origin along,
|
| 519 |
+
# and the anchors follow it, which is stock's arithmetic and not ours.
|
| 520 |
+
assert captured["minimax_keyframes"], "keyframes lost on the R2V path"
|
| 521 |
+
assert audio_kf()["audio_latent"].shape[-1] == 37
|
| 522 |
+
assert len(r2v_cond[0][1]["minimax_refs"]) == 3
|
| 523 |
+
print("R2V path: 3 incoming references untouched, motion context audio "
|
| 524 |
+
"rides as a keyframe, keyframes intact")
|
| 525 |
+
|
| 526 |
+
# last_frame passthrough: an fl2va graph's own anchors used to be
|
| 527 |
+
# replaced outright. The last-frame anchor must survive untouched,
|
| 528 |
+
# because every keyframe now carries its real index and stock
|
| 529 |
+
# compensates them all the same way. The first-frame anchor sits
|
| 530 |
+
# inside the pinned head, which the pinned run already decides, so it
|
| 531 |
+
# is dropped with a warning.
|
| 532 |
+
captured.clear()
|
| 533 |
+
up_kf = [{"resolved_frame_index": 0, "latent": "FF"},
|
| 534 |
+
{"resolved_frame_index": frames - 1, "latent": "LF"}]
|
| 535 |
+
fl_cond = [["c", {"minimax_keyframes": [dict(k) for k in up_kf]}]]
|
| 536 |
+
catcher2 = _Catcher()
|
| 537 |
+
nodes._LOG.addHandler(catcher2)
|
| 538 |
+
try:
|
| 539 |
+
run(
|
| 540 |
+
conditioning=fl_cond, vae=VAE(), latent=target,
|
| 541 |
+
context_frames=context, context_length="22",
|
| 542 |
+
audio_context_length=22, context_latent=prev)
|
| 543 |
+
finally:
|
| 544 |
+
nodes._LOG.removeHandler(catcher2)
|
| 545 |
+
merged = captured["minimax_keyframes"]
|
| 546 |
+
# kept anchor + 7 pinned steps + the pinned audio keyframe
|
| 547 |
+
assert len(merged) == 1 + 7 + 1, len(merged)
|
| 548 |
+
assert merged[0]["latent"] == "LF"
|
| 549 |
+
assert merged[0]["resolved_frame_index"] == frames - 1
|
| 550 |
+
assert merged[0] is not fl_cond[0][1]["minimax_keyframes"][1], \
|
| 551 |
+
"the caller's dict was passed through by reference"
|
| 552 |
+
assert [kf["resolved_frame_index"] for kf in merged[1:8]] == idx
|
| 553 |
+
assert any("dropped 1 keyframe anchor" in m for m in catcher2.msgs), \
|
| 554 |
+
catcher2.msgs
|
| 555 |
+
# an anchor past the end of this clip is a wiring error: the
|
| 556 |
+
# conditioning was resolved against a longer clip, so the anchor would
|
| 557 |
+
# land at the wrong frame. Refuse rather than render it.
|
| 558 |
+
bad_cond = [["c", {"minimax_keyframes": [
|
| 559 |
+
{"resolved_frame_index": frames + 17, "latent": "LF"}]}]]
|
| 560 |
+
try:
|
| 561 |
+
run(conditioning=bad_cond, vae=VAE(), latent=target,
|
| 562 |
+
context_frames=context, context_length="22",
|
| 563 |
+
audio_context_length=22, context_latent=prev)
|
| 564 |
+
except ValueError as e:
|
| 565 |
+
assert "only %d frames" % frames in str(e), str(e)
|
| 566 |
+
else:
|
| 567 |
+
raise AssertionError("out-of-range anchor did not refuse")
|
| 568 |
+
print("last_frame path: upstream last-frame anchor kept unmodified, "
|
| 569 |
+
"first-frame anchor dropped from the pinned head, out-of-range "
|
| 570 |
+
"anchor refused")
|
| 571 |
+
|
| 572 |
+
# save -> load -> context_latent roundtrip across "runs"
|
| 573 |
+
import time
|
| 574 |
+
saver = nodes.MiniMaxH3MotionContextSaveLatent()
|
| 575 |
+
loader = nodes.MiniMaxH3MotionContextLoadLatent()
|
| 576 |
+
(p1,) = saver.save(prev, "h3_context/clip")
|
| 577 |
+
time.sleep(0.02)
|
| 578 |
+
prev2 = {"samples": Nested([
|
| 579 |
+
prev["samples"].parts[0],
|
| 580 |
+
T(prev["samples"].parts[1].a * 2.0), # distinguishable content
|
| 581 |
+
])}
|
| 582 |
+
(p2,) = saver.save(prev2, "h3_context/clip")
|
| 583 |
+
assert p1 != p2
|
| 584 |
+
(loaded,) = loader.load(p2, clip_index=1) # specific file
|
| 585 |
+
parts = loaded["samples"]
|
| 586 |
+
assert isinstance(parts, list) and len(parts) == 2
|
| 587 |
+
captured.clear()
|
| 588 |
+
run(
|
| 589 |
+
conditioning=[["c", {}]], vae=VAE(), latent=target,
|
| 590 |
+
context_frames=context, context_length="22",
|
| 591 |
+
audio_context_length=22, context_latent=loaded)
|
| 592 |
+
kf3 = audio_kf()
|
| 593 |
+
want = float(prev2["samples"].parts[1].a[0, 0, 0, -1])
|
| 594 |
+
got = float(kf3["audio_latent"].a[0, 0, 0, -1])
|
| 595 |
+
assert got == want, (got, want)
|
| 596 |
+
assert abs(audio_end() - 22.2) < 1e-6
|
| 597 |
+
ic1 = loader.IS_CHANGED(p2, clip_index=1)
|
| 598 |
+
assert isinstance(ic1, str) and p2 in ic1 # cache keys on the real file
|
| 599 |
+
assert loader.IS_CHANGED("h3_context", clip_index=0) == "disabled"
|
| 600 |
+
(empty,) = loader.load("h3_context", clip_index=0)
|
| 601 |
+
assert empty is None
|
| 602 |
+
print("save/load roundtrip: file loaded, pinned, end_frame %.4f, "
|
| 603 |
+
"index 0 is first-clip/no-context" % audio_end())
|
| 604 |
+
|
| 605 |
+
# retry safety with indexed slots: generating clip 3, re-rolling it
|
| 606 |
+
# must overwrite slot 3 and always load slot 2, never its own save
|
| 607 |
+
prevA = {"samples": Nested([prev["samples"].parts[0],
|
| 608 |
+
T(np.full((1, 32, 2, audio_t), 7.0,
|
| 609 |
+
dtype=np.float32))])}
|
| 610 |
+
prevB1 = {"samples": Nested([prev["samples"].parts[0],
|
| 611 |
+
T(np.full((1, 32, 2, audio_t), 8.0,
|
| 612 |
+
dtype=np.float32))])}
|
| 613 |
+
prevB2 = {"samples": Nested([prev["samples"].parts[0],
|
| 614 |
+
T(np.full((1, 32, 2, audio_t), 9.0,
|
| 615 |
+
dtype=np.float32))])}
|
| 616 |
+
(pa,) = saver.save(prevA, "h3_context/clip", clip_index=2) # clip 2 ok
|
| 617 |
+
assert pa.endswith("_00002.safetensors"), pa # natural slot name
|
| 618 |
+
time.sleep(0.02)
|
| 619 |
+
(pb1,) = saver.save(prevB1, "h3_context/clip", clip_index=3) # clip 3 try 1
|
| 620 |
+
time.sleep(0.02)
|
| 621 |
+
(pb2,) = saver.save(prevB2, "h3_context/clip", clip_index=3) # re-roll
|
| 622 |
+
assert pb1 == pb2 and pa != pb1 # re-roll overwrote its own slot
|
| 623 |
+
# generating clip 3, continuing FROM clip 2: loader index is 2, literally
|
| 624 |
+
(l3,) = loader.load("h3_context", clip_index=2)
|
| 625 |
+
got = float(l3["samples"][1].a[0, 0, 0, 0])
|
| 626 |
+
assert got == 7.0, got # clip 2's latent, NOT the rejected attempt (8/9)
|
| 627 |
+
# index 0 is first-clip/no-context: it must not pick up the reject
|
| 628 |
+
(lnew,) = loader.load("h3_context", clip_index=0)
|
| 629 |
+
assert lnew is None
|
| 630 |
+
# asking for a slot that was never saved says so plainly
|
| 631 |
+
try:
|
| 632 |
+
loader.load("h3_context", clip_index=7)
|
| 633 |
+
except FileNotFoundError as e:
|
| 634 |
+
assert "no saved latent for clip 7" in str(e)
|
| 635 |
+
else:
|
| 636 |
+
raise AssertionError("missing slot did not refuse")
|
| 637 |
+
# an auto-numbered near-miss (trailing underscore) is never matched,
|
| 638 |
+
# and the error explains the rename
|
| 639 |
+
(pauto,) = saver.save(prevA, "h3_context/clip", clip_index=0)
|
| 640 |
+
assert pauto.endswith("_.safetensors"), pauto
|
| 641 |
+
import re as _re
|
| 642 |
+
runno = int(_re.search(r"_(\d{5})_\.safetensors$", pauto).group(1))
|
| 643 |
+
try:
|
| 644 |
+
loader.load("h3_context", clip_index=runno)
|
| 645 |
+
except FileNotFoundError as e:
|
| 646 |
+
assert "trailing underscore" in str(e) and "rename" in str(e), str(e)
|
| 647 |
+
else:
|
| 648 |
+
raise AssertionError("auto-numbered file was matched by index")
|
| 649 |
+
print("indexed slots: re-roll overwrites its slot, loads previous "
|
| 650 |
+
"clip's latent; auto mode confirmed to return the reject")
|
| 651 |
+
|
| 652 |
+
import shutil
|
| 653 |
+
td = os.path.join(outdir, "clear_test")
|
| 654 |
+
os.makedirs(td)
|
| 655 |
+
try:
|
| 656 |
+
open(os.path.join(td, "clip_00001.safetensors"), "wb").close()
|
| 657 |
+
open(os.path.join(td, "clip_00002.safetensors"), "wb").close()
|
| 658 |
+
open(os.path.join(td, "hero_take.safetensors"), "wb").close()
|
| 659 |
+
open(os.path.join(td, "keep_me.txt"), "w").write("x")
|
| 660 |
+
assert nodes._clip_slot_exists(td, 1)
|
| 661 |
+
assert nodes._clip_slot_exists(td, 2)
|
| 662 |
+
assert not nodes._clip_slot_exists(td, 3)
|
| 663 |
+
n = nodes._clear_clip_slots(td)
|
| 664 |
+
assert n == 2, n
|
| 665 |
+
assert not nodes._clip_slot_exists(td, 1)
|
| 666 |
+
assert os.path.exists(os.path.join(td, "keep_me.txt"))
|
| 667 |
+
assert os.path.exists(os.path.join(td, "hero_take.safetensors"))
|
| 668 |
+
finally:
|
| 669 |
+
shutil.rmtree(td)
|
| 670 |
+
print("reset clear: numbered slots gone, custom names left alone")
|
| 671 |
+
|
| 672 |
+
print("smoke test passed")
|
| 673 |
+
|
| 674 |
+
|
| 675 |
+
if __name__ == "__main__":
|
| 676 |
+
main()
|
custom_nodes/ComfyUI-H3-Motion-Context-main/tests/_probe_node_test.py
ADDED
|
@@ -0,0 +1,356 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Probe node test: drive the seam probe with joins whose answer is known.
|
| 2 |
+
|
| 3 |
+
No ComfyUI and no GPU. Builds audio where the continuation, the lag and
|
| 4 |
+
the level step are all constructed, then checks the node reports them.
|
| 5 |
+
Clip A goes in as a latent, so the fake audio VAE's decode is what the
|
| 6 |
+
node reads, exactly as it would in a graph.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
import os
|
| 10 |
+
import sys
|
| 11 |
+
import types
|
| 12 |
+
|
| 13 |
+
import numpy as np
|
| 14 |
+
|
| 15 |
+
_TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 16 |
+
_PKG_DIR = os.path.dirname(_TESTS_DIR)
|
| 17 |
+
sys.path.insert(0, _TESTS_DIR)
|
| 18 |
+
|
| 19 |
+
from _mock_harness import make_mm, make_torch # noqa: E402
|
| 20 |
+
|
| 21 |
+
SR = 32000
|
| 22 |
+
FPS = 24.0
|
| 23 |
+
TRIM = 22
|
| 24 |
+
SPAN = int(round(TRIM / FPS * SR))
|
| 25 |
+
# a legal clip length on the ladder: 5g+2 latent steps, 17g+5 frames
|
| 26 |
+
LATENT_T, FRAMES = 37, 124
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _install_fakes():
|
| 30 |
+
"""nodes.py imports ComfyUI, and the probe imports nodes.py.
|
| 31 |
+
|
| 32 |
+
Only enough of each module to get through import: this test never
|
| 33 |
+
runs the other nodes, so the fakes do not have to be faithful the way
|
| 34 |
+
the smoke test's are.
|
| 35 |
+
"""
|
| 36 |
+
mm = make_mm()
|
| 37 |
+
for name in ("comfy", "comfy.ldm", "comfy.ldm.minimax"):
|
| 38 |
+
sys.modules.setdefault(name, types.ModuleType(name))
|
| 39 |
+
sys.modules["comfy.ldm.minimax.model"] = mm
|
| 40 |
+
sys.modules["comfy"].ldm = sys.modules["comfy.ldm"]
|
| 41 |
+
sys.modules["comfy.ldm"].minimax = sys.modules["comfy.ldm.minimax"]
|
| 42 |
+
sys.modules["comfy.ldm.minimax"].model = mm
|
| 43 |
+
sys.modules["torch"] = make_torch()
|
| 44 |
+
|
| 45 |
+
# safetensors must be faked even though nodes.py never uses it here:
|
| 46 |
+
# the real package imports torch at module scope, and with the fake
|
| 47 |
+
# torch in sys.modules that dies with an AttributeError the ImportError
|
| 48 |
+
# guard in nodes.py does not catch
|
| 49 |
+
for name, attrs in (("comfy.utils", {}),
|
| 50 |
+
("comfy.model_base", {"MiniMaxH3": type(
|
| 51 |
+
"MiniMaxH3", (), {"extra_conds": lambda s, **k: {}})}),
|
| 52 |
+
("folder_paths", {"get_output_directory": lambda: ".",
|
| 53 |
+
"get_save_image_path":
|
| 54 |
+
lambda *a: (".", "x", 1, "", "")}),
|
| 55 |
+
("node_helpers", {"conditioning_set_values":
|
| 56 |
+
lambda c, v, **k: c}),
|
| 57 |
+
("safetensors", {}),
|
| 58 |
+
("safetensors.torch", {"load_file": None,
|
| 59 |
+
"save_file": None})):
|
| 60 |
+
mod = types.ModuleType(name)
|
| 61 |
+
for k, v in attrs.items():
|
| 62 |
+
setattr(mod, k, v)
|
| 63 |
+
sys.modules[name] = mod
|
| 64 |
+
if name.startswith("comfy."):
|
| 65 |
+
setattr(sys.modules["comfy"], name.split(".", 1)[1], mod)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
_install_fakes()
|
| 69 |
+
# Relative imports (from .nodes) need a package. The folder name has
|
| 70 |
+
# hyphens, so a stub name is used instead of a dynamic import.
|
| 71 |
+
_pkg = types.ModuleType("h3mc_pkg")
|
| 72 |
+
_pkg.__path__ = [_PKG_DIR]
|
| 73 |
+
_pkg.__file__ = os.path.join(_PKG_DIR, "__init__.py")
|
| 74 |
+
sys.modules["h3mc_pkg"] = _pkg
|
| 75 |
+
import h3mc_pkg.probe_node as probe_node # noqa: E402
|
| 76 |
+
CORR_LIMIT = probe_node.CORR_CREDIBLE
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
class Tensor:
|
| 80 |
+
"""Enough of a tensor for the node's shape checks and the fake VAE."""
|
| 81 |
+
|
| 82 |
+
def __init__(self, a):
|
| 83 |
+
self.a = np.asarray(a)
|
| 84 |
+
|
| 85 |
+
@property
|
| 86 |
+
def shape(self):
|
| 87 |
+
return self.a.shape
|
| 88 |
+
|
| 89 |
+
@property
|
| 90 |
+
def ndim(self):
|
| 91 |
+
return self.a.ndim
|
| 92 |
+
|
| 93 |
+
def __getitem__(self, idx):
|
| 94 |
+
return Tensor(self.a[idx])
|
| 95 |
+
|
| 96 |
+
def unsqueeze(self, d):
|
| 97 |
+
return Tensor(np.expand_dims(self.a, d))
|
| 98 |
+
|
| 99 |
+
def __array__(self, dtype=None, copy=None):
|
| 100 |
+
# real torch tensors implement this, which is how np.asarray()
|
| 101 |
+
# accepts one; without it numpy falls back to iterating the
|
| 102 |
+
# object and the conversion fails
|
| 103 |
+
return self.a if dtype is None else self.a.astype(dtype)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
class Nested:
|
| 107 |
+
def __init__(self, parts):
|
| 108 |
+
self.parts = parts
|
| 109 |
+
|
| 110 |
+
def unbind(self):
|
| 111 |
+
return list(self.parts)
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
class FakeAudioVAE:
|
| 115 |
+
"""Decodes by handing back the waveform stashed on it.
|
| 116 |
+
|
| 117 |
+
The point of the test is the node's anchoring and measurement, not
|
| 118 |
+
the VAE, so the decode is a lookup. It returns [B, C, L], which the
|
| 119 |
+
node must not assume the orientation of.
|
| 120 |
+
"""
|
| 121 |
+
|
| 122 |
+
audio_sample_rate = SR
|
| 123 |
+
|
| 124 |
+
def __init__(self, mono):
|
| 125 |
+
self.mono = mono
|
| 126 |
+
|
| 127 |
+
def decode(self, z):
|
| 128 |
+
return Tensor(np.stack([self.mono, self.mono])[None, ...])
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def _latent(mono, latent_t=LATENT_T):
|
| 132 |
+
"""An AV latent whose video stream says how many frames the clip is."""
|
| 133 |
+
steps = max(1, int(round(len(mono) / SR * 40)))
|
| 134 |
+
return {"samples": Nested([
|
| 135 |
+
Tensor(np.zeros((1, 16, latent_t, 4, 4), dtype=np.float32)),
|
| 136 |
+
Tensor(np.zeros((1, 32, 2, steps), dtype=np.float32)),
|
| 137 |
+
])}
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def _audio(mono):
|
| 141 |
+
return {"waveform": np.stack([mono, mono])[None, ...],
|
| 142 |
+
"sample_rate": SR}
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def _bed(n, seed, freq=110.0, amp=1.0, noise=0.02):
|
| 146 |
+
"""Periodic content plus a steady floor. The tone is deliberately
|
| 147 |
+
periodic: it is what the peak tracker has to avoid cycle-hopping on."""
|
| 148 |
+
rng = np.random.default_rng(seed)
|
| 149 |
+
t = np.arange(n) / SR
|
| 150 |
+
return (amp * np.sin(2 * np.pi * freq * t)
|
| 151 |
+
+ 0.3 * amp * np.sin(2 * np.pi * freq * 2.5 * t)
|
| 152 |
+
+ noise * rng.standard_normal(n))
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def _gated(n, seed, floor_amp, freq=110.0, on_ms=120.0, off_ms=80.0):
|
| 156 |
+
"""Tone bursts over a continuous noise bed.
|
| 157 |
+
|
| 158 |
+
The floor metric reads the quiet moments between content, so a fully
|
| 159 |
+
sustained tone gives it nothing and it collapses onto broadband. That
|
| 160 |
+
limitation is documented in level_step.py, so the fixture for a floor
|
| 161 |
+
step has to have rests in it.
|
| 162 |
+
"""
|
| 163 |
+
rng = np.random.default_rng(seed)
|
| 164 |
+
t = np.arange(n) / SR
|
| 165 |
+
tone = np.sin(2 * np.pi * freq * t)
|
| 166 |
+
period = int(round((on_ms + off_ms) / 1000.0 * SR))
|
| 167 |
+
on = int(round(on_ms / 1000.0 * SR))
|
| 168 |
+
gate = (np.arange(n) % period) < on
|
| 169 |
+
return tone * gate + floor_amp * rng.standard_normal(n)
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def _run(a_mono, b_mono, **kw):
|
| 173 |
+
node = probe_node.MiniMaxH3MotionContextSeamProbe()
|
| 174 |
+
out = node.probe(clip_b_untrimmed=_audio(b_mono), trim_frames=TRIM,
|
| 175 |
+
clip_a_latent=_latent(a_mono),
|
| 176 |
+
audio_vae=FakeAudioVAE(a_mono), fps=FPS, **kw)
|
| 177 |
+
return out["result"][0], out["result"][1]
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def _num(report, label):
|
| 181 |
+
for line in report.splitlines():
|
| 182 |
+
s = line.strip()
|
| 183 |
+
if s.startswith(label):
|
| 184 |
+
for tok in s[len(label):].split():
|
| 185 |
+
try:
|
| 186 |
+
return float(tok)
|
| 187 |
+
except ValueError:
|
| 188 |
+
continue
|
| 189 |
+
raise AssertionError("no %r line in:\n%s" % (label, report))
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def main():
|
| 193 |
+
# clip A is exactly FRAMES long, which is what the latent declares.
|
| 194 |
+
# The node tail-matches to that, so the fixtures must agree.
|
| 195 |
+
a_len = int(round(FRAMES / FPS * SR))
|
| 196 |
+
source = _bed(a_len + SPAN * 3, seed=1)
|
| 197 |
+
a = source[:a_len]
|
| 198 |
+
|
| 199 |
+
# 1. perfect continuation: B's head IS A's tail
|
| 200 |
+
b = np.concatenate([a[-SPAN:], source[a_len:a_len + SPAN * 2]])
|
| 201 |
+
passthru, rep = _run(a, b)
|
| 202 |
+
corr, lag = _num(rep, "mean corr"), _num(rep, "lag")
|
| 203 |
+
assert corr > 0.95, (corr, rep)
|
| 204 |
+
assert abs(lag) < 1.0, (lag, rep)
|
| 205 |
+
assert "clean continuation" in rep, rep
|
| 206 |
+
assert passthru["sample_rate"] == SR
|
| 207 |
+
print("1. exact continuation: corr %.3f, lag %+.2f ms" % (corr, lag))
|
| 208 |
+
|
| 209 |
+
# 2. late by one audio step
|
| 210 |
+
shift = int(round(0.025 * SR))
|
| 211 |
+
b_late = np.concatenate([source[a_len - SPAN - shift:a_len - shift],
|
| 212 |
+
source[a_len:a_len + SPAN * 2]])
|
| 213 |
+
_, rep = _run(a, b_late)
|
| 214 |
+
corr, lag = _num(rep, "mean corr"), _num(rep, "lag")
|
| 215 |
+
assert corr > 0.9, (corr, rep)
|
| 216 |
+
assert abs(lag - 25.0) < 2.0, (lag, rep)
|
| 217 |
+
print("2. one step late: corr %.3f, lag %+.2f ms (built as +25.0)"
|
| 218 |
+
% (corr, lag))
|
| 219 |
+
|
| 220 |
+
# 3. imitation, not continuation. A different noise seed alone is not
|
| 221 |
+
# enough, the deterministic tone would still correlate at 0.99.
|
| 222 |
+
b_fake = np.concatenate([_bed(SPAN, seed=99, freq=173.0),
|
| 223 |
+
source[a_len:a_len + SPAN * 2]])
|
| 224 |
+
_, rep = _run(a, b_fake)
|
| 225 |
+
corr = _num(rep, "mean corr")
|
| 226 |
+
assert corr < CORR_LIMIT, (corr, rep)
|
| 227 |
+
assert "imitating" in rep or "not being continued" in rep, rep
|
| 228 |
+
print("3. unrelated head: corr %.3f, flagged as imitation" % corr)
|
| 229 |
+
|
| 230 |
+
# 4. same music either side, but B's bed is 30 dB quieter. Invisible
|
| 231 |
+
# to broadband because the content matches.
|
| 232 |
+
gated_a = _gated(a_len, seed=11, floor_amp=0.02)
|
| 233 |
+
loud = np.concatenate([gated_a[-SPAN:],
|
| 234 |
+
_gated(SPAN * 2, seed=12, floor_amp=0.02)])
|
| 235 |
+
quiet = np.concatenate([gated_a[-SPAN:],
|
| 236 |
+
_gated(SPAN * 2, seed=12, floor_amp=0.0006)])
|
| 237 |
+
_, rep = _run(gated_a, quiet)
|
| 238 |
+
floor, bb = _num(rep, "floor"), _num(rep, "broadband")
|
| 239 |
+
assert floor > 0.5, (floor, rep)
|
| 240 |
+
assert bb < 0.4, (bb, rep)
|
| 241 |
+
assert "bed underneath" in rep, rep
|
| 242 |
+
print("4. floor drop: floor %.3f (audible), broadband %.3f (ok), "
|
| 243 |
+
"diagnosed as ambience restart" % (floor, bb))
|
| 244 |
+
|
| 245 |
+
# 5. no level step when both sides share a bed
|
| 246 |
+
_, rep = _run(gated_a, loud)
|
| 247 |
+
floor = _num(rep, "floor")
|
| 248 |
+
assert floor < 0.25, (floor, rep)
|
| 249 |
+
assert "bed underneath" not in rep, rep
|
| 250 |
+
print("5. matched bed: floor step %.3f, below the clean threshold"
|
| 251 |
+
% floor)
|
| 252 |
+
|
| 253 |
+
# 6. a lag outside the search window must not become a confident
|
| 254 |
+
# number. On non-periodic content nothing correlates, so no lag.
|
| 255 |
+
rng = np.random.default_rng(21)
|
| 256 |
+
na = rng.standard_normal(a_len)
|
| 257 |
+
big = int(round(0.060 * SR))
|
| 258 |
+
nb = np.concatenate([na[-SPAN - big:-big],
|
| 259 |
+
rng.standard_normal(SPAN * 2)])
|
| 260 |
+
_, rep = _run(na, nb, search_ms=20.0)
|
| 261 |
+
assert "not reported" in rep, rep
|
| 262 |
+
assert _num(rep, "mean corr") < CORR_LIMIT, rep
|
| 263 |
+
print("6. lag outside the search window: no lag reported rather than "
|
| 264 |
+
"a confident wrong one")
|
| 265 |
+
|
| 266 |
+
# 7. the aliasing limitation, asserted rather than left implicit.
|
| 267 |
+
# Beat-driven music is exactly this case.
|
| 268 |
+
period_ms = 1000.0 / 55.0 # 110 and 275 Hz share a 55 Hz fundamental
|
| 269 |
+
b_alias = np.concatenate([source[a_len - SPAN - big:a_len - big],
|
| 270 |
+
source[a_len:a_len + SPAN * 2]])
|
| 271 |
+
_, rep = _run(a, b_alias, search_ms=20.0)
|
| 272 |
+
lag = _num(rep, "lag")
|
| 273 |
+
assert _num(rep, "mean corr") > 0.9, rep
|
| 274 |
+
assert abs(lag - 60.0) > 10.0, (lag, rep)
|
| 275 |
+
assert abs(((60.0 - lag) / period_ms)
|
| 276 |
+
- round((60.0 - lag) / period_ms)) < 0.05, (lag, rep)
|
| 277 |
+
print("7. periodic aliasing: 60.0 ms offset read as %+.2f ms, a whole "
|
| 278 |
+
"number of cycles out (known limitation)" % lag)
|
| 279 |
+
|
| 280 |
+
# 8. the node must tail-match clip A itself. Hand it a decode that is
|
| 281 |
+
# a third of an audio step long, the real grid overhang, and the
|
| 282 |
+
# lag must stay at zero. Without the internal match this reads as
|
| 283 |
+
# a confident 8.3 ms, which is exactly the artifact the node
|
| 284 |
+
# exists to remove.
|
| 285 |
+
over = int(round(SR / 40.0 / 3.0))
|
| 286 |
+
a_long = np.concatenate([a, source[a_len:a_len + over]])
|
| 287 |
+
node = probe_node.MiniMaxH3MotionContextSeamProbe()
|
| 288 |
+
out = node.probe(clip_b_untrimmed=_audio(b), trim_frames=TRIM,
|
| 289 |
+
clip_a_latent=_latent(a), # declares FRAMES frames
|
| 290 |
+
audio_vae=FakeAudioVAE(a_long), fps=FPS)
|
| 291 |
+
lag = _num(out["result"][1], "lag")
|
| 292 |
+
assert abs(lag) < 1.0, (lag, out["result"][1])
|
| 293 |
+
print("8. overlong decode (%d samples, %.2f ms): trimmed internally, "
|
| 294 |
+
"lag stays %+.2f ms" % (over, over / SR * 1000.0, lag))
|
| 295 |
+
|
| 296 |
+
# 9. wrong wiring refused with a reason, audio still passed through
|
| 297 |
+
node = probe_node.MiniMaxH3MotionContextSeamProbe()
|
| 298 |
+
cases = [
|
| 299 |
+
("TRIMMED", dict(clip_b_untrimmed=_audio(b[SPAN:][:SPAN // 2]),
|
| 300 |
+
trim_frames=TRIM, clip_a_latent=_latent(a),
|
| 301 |
+
audio_vae=FakeAudioVAE(a))),
|
| 302 |
+
("trim_frames is 0", dict(clip_b_untrimmed=_audio(b),
|
| 303 |
+
trim_frames=0, clip_a_latent=_latent(a),
|
| 304 |
+
audio_vae=FakeAudioVAE(a))),
|
| 305 |
+
("no clip_a_latent", dict(clip_b_untrimmed=_audio(b),
|
| 306 |
+
trim_frames=TRIM)),
|
| 307 |
+
("no audio_vae", dict(clip_b_untrimmed=_audio(b),
|
| 308 |
+
trim_frames=TRIM,
|
| 309 |
+
clip_a_latent=_latent(a))),
|
| 310 |
+
]
|
| 311 |
+
for needle, kwargs in cases:
|
| 312 |
+
res = node.probe(fps=FPS, **kwargs)
|
| 313 |
+
assert needle in res["result"][1], (needle, res["result"][1])
|
| 314 |
+
assert res["result"][0]["sample_rate"] == SR
|
| 315 |
+
vid_only = {"samples": Nested([
|
| 316 |
+
Tensor(np.zeros((1, 16, LATENT_T, 4, 4), dtype=np.float32))])}
|
| 317 |
+
res = node.probe(clip_b_untrimmed=_audio(b), trim_frames=TRIM,
|
| 318 |
+
clip_a_latent=vid_only, audio_vae=FakeAudioVAE(a),
|
| 319 |
+
fps=FPS)
|
| 320 |
+
assert "no audio stream" in res["result"][1], res["result"][1]
|
| 321 |
+
print("9. bad wiring: trimmed input, zero span, missing latent, "
|
| 322 |
+
"missing VAE and video-only latent each refused with a reason, "
|
| 323 |
+
"audio still passed through")
|
| 324 |
+
|
| 325 |
+
# 10. a SHORT clip A gets padded, and padding moves the end of the
|
| 326 |
+
# array without moving the content. So the padded anchor and the
|
| 327 |
+
# raw one disagree by exactly the pad, and the node must report
|
| 328 |
+
# both rather than pick one. Built here so the raw anchor is the
|
| 329 |
+
# correct one: B's head is A's real tail.
|
| 330 |
+
short = int(round(SR / 40.0 / 3.0))
|
| 331 |
+
a_short = a[:a_len - short]
|
| 332 |
+
b_short = np.concatenate([a_short[-SPAN:],
|
| 333 |
+
source[a_len:a_len + SPAN * 2]])
|
| 334 |
+
node = probe_node.MiniMaxH3MotionContextSeamProbe()
|
| 335 |
+
out = node.probe(clip_b_untrimmed=_audio(b_short), trim_frames=TRIM,
|
| 336 |
+
clip_a_latent=_latent(a), # declares FRAMES frames
|
| 337 |
+
audio_vae=FakeAudioVAE(a_short), fps=FPS)
|
| 338 |
+
rep = out["result"][1]
|
| 339 |
+
assert "cross-check" in rep, rep
|
| 340 |
+
assert "padded %d samples" % short in rep, rep
|
| 341 |
+
lags = [float(l.strip()[len("lag"):].split()[0])
|
| 342 |
+
for l in rep.splitlines() if l.strip().startswith("lag")]
|
| 343 |
+
assert len(lags) == 2, (lags, rep)
|
| 344 |
+
padded_lag, raw_lag = lags
|
| 345 |
+
assert abs(raw_lag) < 1.0, (raw_lag, rep)
|
| 346 |
+
assert abs(abs(padded_lag - raw_lag) - short / SR * 1000.0) < 1.0, \
|
| 347 |
+
(padded_lag, raw_lag, rep)
|
| 348 |
+
print("10. short clip A: both anchors reported, %+.2f ms padded vs "
|
| 349 |
+
"%+.2f ms raw, differing by the %.2f ms pad"
|
| 350 |
+
% (padded_lag, raw_lag, short / SR * 1000.0))
|
| 351 |
+
|
| 352 |
+
print("probe node test passed")
|
| 353 |
+
|
| 354 |
+
|
| 355 |
+
if __name__ == "__main__":
|
| 356 |
+
main()
|
custom_nodes/ComfyUI-H3-Motion-Context-main/tests/freeze_detect.py
ADDED
|
@@ -0,0 +1,286 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Freeze probe: did a shot render as a still instead of a held frame?
|
| 2 |
+
|
| 3 |
+
The airlock technique for chaining across a setup change asks clip N+1 to
|
| 4 |
+
open by holding clip N's exact closing framing for a beat before cutting
|
| 5 |
+
to the new setup. That works, but a held framing with nothing happening
|
| 6 |
+
in it renders as a literal freeze: several seconds of a motionless actor
|
| 7 |
+
that reads as a stalled video player, not a pause. It is easy to miss on
|
| 8 |
+
a first watch and obvious once someone points at it.
|
| 9 |
+
|
| 10 |
+
This walks a clip and reports every stretch where the picture stops
|
| 11 |
+
moving.
|
| 12 |
+
|
| 13 |
+
Usage:
|
| 14 |
+
python freeze_detect.py clip01.mp4 clip02.mp4 ...
|
| 15 |
+
python freeze_detect.py --head-frames 22 --min-ms 400 clip02.mp4
|
| 16 |
+
|
| 17 |
+
Method: decode, downscale each frame to a small grey thumbnail, and take
|
| 18 |
+
the mean absolute difference between consecutive thumbnails. Downscaling
|
| 19 |
+
is what makes this work on compressed video, since it averages away the
|
| 20 |
+
codec noise that would otherwise register as motion in an otherwise
|
| 21 |
+
static frame.
|
| 22 |
+
|
| 23 |
+
The threshold is relative, not absolute. An absolute one cannot serve
|
| 24 |
+
both a dim interior and a bright exterior, so each clip is scored against
|
| 25 |
+
its own median frame difference and a stretch is flagged when motion
|
| 26 |
+
drops below `--rel` of that median (default 0.15) for at least `--min-ms`
|
| 27 |
+
(default 400). A separate absolute floor catches duplicated frames that
|
| 28 |
+
are identical to within rounding.
|
| 29 |
+
|
| 30 |
+
The first `--head-frames` are reported separately whatever they score,
|
| 31 |
+
because on a chained clip those are the pinned head: the model's re-render
|
| 32 |
+
of the previous clip's tail, which is the region the trim removes and the
|
| 33 |
+
region an airlock hold lives in. Motion there that is far below the rest
|
| 34 |
+
of the clip means the hold had no business in it.
|
| 35 |
+
|
| 36 |
+
Reading the result:
|
| 37 |
+
no runs nothing held
|
| 38 |
+
a run inside the head the airlock hold has nothing happening in
|
| 39 |
+
it. Give it a breath, a weight shift, an
|
| 40 |
+
eyeline change. The framing holds; the
|
| 41 |
+
performer should not.
|
| 42 |
+
a run elsewhere the prompt ran out of things for the shot
|
| 43 |
+
to do, or the clip is longer than the
|
| 44 |
+
action written for it
|
| 45 |
+
|
| 46 |
+
Dependencies: numpy, plus ONE of av / cv2 / imageio for decoding.
|
| 47 |
+
ComfyUI's embedded python has numpy and PyAV; run it with that
|
| 48 |
+
interpreter if your system python lacks them.
|
| 49 |
+
"""
|
| 50 |
+
|
| 51 |
+
import argparse
|
| 52 |
+
import os
|
| 53 |
+
import sys
|
| 54 |
+
|
| 55 |
+
import numpy as np
|
| 56 |
+
|
| 57 |
+
THUMB_W, THUMB_H = 64, 36
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _to_thumb(rgb):
|
| 61 |
+
"""[H, W, 3] uint8 -> [THUMB_H, THUMB_W] float64 grey, box averaged.
|
| 62 |
+
|
| 63 |
+
Box averaging rather than nearest sampling, because nearest keeps the
|
| 64 |
+
per-pixel codec noise this is trying to average away.
|
| 65 |
+
"""
|
| 66 |
+
a = np.asarray(rgb)
|
| 67 |
+
if a.ndim == 3 and a.shape[2] >= 3:
|
| 68 |
+
grey = a[..., :3].astype(np.float64) @ np.array([0.299, 0.587, 0.114])
|
| 69 |
+
else:
|
| 70 |
+
grey = a.astype(np.float64).squeeze()
|
| 71 |
+
h, w = grey.shape
|
| 72 |
+
ys = np.linspace(0, h, THUMB_H + 1).astype(int)
|
| 73 |
+
xs = np.linspace(0, w, THUMB_W + 1).astype(int)
|
| 74 |
+
out = np.empty((THUMB_H, THUMB_W), dtype=np.float64)
|
| 75 |
+
for i in range(THUMB_H):
|
| 76 |
+
y0, y1 = ys[i], max(ys[i] + 1, ys[i + 1])
|
| 77 |
+
row = grey[y0:y1]
|
| 78 |
+
for j in range(THUMB_W):
|
| 79 |
+
x0, x1 = xs[j], max(xs[j] + 1, xs[j + 1])
|
| 80 |
+
out[i, j] = row[:, x0:x1].mean()
|
| 81 |
+
return out
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def iter_thumbs(path):
|
| 85 |
+
"""Yield (thumbnail, fps). Tries PyAV, then OpenCV, then imageio."""
|
| 86 |
+
try:
|
| 87 |
+
import av
|
| 88 |
+
with av.open(path) as container:
|
| 89 |
+
stream = container.streams.video[0]
|
| 90 |
+
fps = float(stream.average_rate) if stream.average_rate else 24.0
|
| 91 |
+
for frame in container.decode(stream):
|
| 92 |
+
yield _to_thumb(frame.to_ndarray(format="rgb24")), fps
|
| 93 |
+
return
|
| 94 |
+
except ImportError:
|
| 95 |
+
pass
|
| 96 |
+
|
| 97 |
+
try:
|
| 98 |
+
import cv2
|
| 99 |
+
cap = cv2.VideoCapture(path)
|
| 100 |
+
if not cap.isOpened():
|
| 101 |
+
raise RuntimeError("cv2 could not open %s" % path)
|
| 102 |
+
fps = cap.get(cv2.CAP_PROP_FPS) or 24.0
|
| 103 |
+
while True:
|
| 104 |
+
ok, frame = cap.read()
|
| 105 |
+
if not ok:
|
| 106 |
+
break
|
| 107 |
+
yield _to_thumb(frame[..., ::-1]), float(fps)
|
| 108 |
+
cap.release()
|
| 109 |
+
return
|
| 110 |
+
except ImportError:
|
| 111 |
+
pass
|
| 112 |
+
|
| 113 |
+
try:
|
| 114 |
+
import imageio.v3 as iio
|
| 115 |
+
meta = iio.immeta(path)
|
| 116 |
+
fps = float(meta.get("fps", 24.0))
|
| 117 |
+
for frame in iio.imiter(path):
|
| 118 |
+
yield _to_thumb(frame), fps
|
| 119 |
+
return
|
| 120 |
+
except ImportError:
|
| 121 |
+
pass
|
| 122 |
+
|
| 123 |
+
raise RuntimeError(
|
| 124 |
+
"no video decoder available. Install one of av, opencv-python or "
|
| 125 |
+
"imageio, or run this with ComfyUI's embedded python which ships "
|
| 126 |
+
"PyAV.")
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def frame_deltas(path):
|
| 130 |
+
"""Mean absolute difference between consecutive frames, 0..255 scale."""
|
| 131 |
+
deltas, prev, fps = [], None, 24.0
|
| 132 |
+
for thumb, f in iter_thumbs(path):
|
| 133 |
+
fps = f
|
| 134 |
+
if prev is not None:
|
| 135 |
+
deltas.append(float(np.abs(thumb - prev).mean()))
|
| 136 |
+
prev = thumb
|
| 137 |
+
return np.array(deltas), fps
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def find_runs(deltas, fps, rel=0.15, abs_floor=0.15, min_ms=400.0):
|
| 141 |
+
"""Stretches where motion sits below the clip's own normal.
|
| 142 |
+
|
| 143 |
+
Returns a list of (start_frame, end_frame_exclusive, mean_delta,
|
| 144 |
+
mean_ratio). `deltas[i]` is the motion between frame i and i+1, so a
|
| 145 |
+
run over delta indices [a, b) means frames a..b are the still ones.
|
| 146 |
+
"""
|
| 147 |
+
if len(deltas) < 2:
|
| 148 |
+
return []
|
| 149 |
+
median = float(np.median(deltas))
|
| 150 |
+
if median <= 1e-9:
|
| 151 |
+
# the entire clip is static; report it as one run rather than
|
| 152 |
+
# dividing by a zero normal
|
| 153 |
+
return [(0, len(deltas) + 1, float(deltas.mean()), 0.0)]
|
| 154 |
+
ratio = deltas / median
|
| 155 |
+
still = (ratio < rel) | (deltas < abs_floor)
|
| 156 |
+
min_frames = max(1, int(round(min_ms / 1000.0 * fps)))
|
| 157 |
+
|
| 158 |
+
runs, start = [], None
|
| 159 |
+
for i, s in enumerate(list(still) + [False]):
|
| 160 |
+
if s and start is None:
|
| 161 |
+
start = i
|
| 162 |
+
elif not s and start is not None:
|
| 163 |
+
if i - start >= min_frames:
|
| 164 |
+
seg = deltas[start:i]
|
| 165 |
+
runs.append((start, i + 1, float(seg.mean()),
|
| 166 |
+
float((seg / median).mean())))
|
| 167 |
+
start = None
|
| 168 |
+
return runs
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def report(path, head_frames, rel, abs_floor, min_ms):
|
| 172 |
+
deltas, fps = frame_deltas(path)
|
| 173 |
+
name = os.path.basename(path)
|
| 174 |
+
if len(deltas) < 2:
|
| 175 |
+
print("%-28s too few frames to score" % name)
|
| 176 |
+
return 0
|
| 177 |
+
median = float(np.median(deltas))
|
| 178 |
+
runs = find_runs(deltas, fps, rel, abs_floor, min_ms)
|
| 179 |
+
print("%-28s %d frames @ %.3f fps, median motion %.3f" %
|
| 180 |
+
(name, len(deltas) + 1, fps, median))
|
| 181 |
+
|
| 182 |
+
if head_frames > 0 and len(deltas) > head_frames:
|
| 183 |
+
head = deltas[:head_frames]
|
| 184 |
+
h_ratio = float(head.mean()) / median if median > 0 else 0.0
|
| 185 |
+
flag = " <- HELD, give it something to do" if h_ratio < rel else ""
|
| 186 |
+
print(" head (first %d frames): motion %.3f, %.2fx the clip "
|
| 187 |
+
"median%s" % (head_frames, float(head.mean()), h_ratio, flag))
|
| 188 |
+
|
| 189 |
+
if not runs:
|
| 190 |
+
print(" no freezes")
|
| 191 |
+
return 0
|
| 192 |
+
for a, b, mean_d, mean_r in runs:
|
| 193 |
+
# deltas[i] is the motion between frame i and frame i+1, so a run
|
| 194 |
+
# over frames a..b-1 holds still for b-1-a intervals
|
| 195 |
+
secs = (b - 1 - a) / fps
|
| 196 |
+
where = "head" if a < head_frames else "body"
|
| 197 |
+
print(" FREEZE %6.2fs to %6.2fs (%5.2fs, %s) motion %.3f "
|
| 198 |
+
"= %.2fx median" %
|
| 199 |
+
(a / fps, (b - 1) / fps, secs, where, mean_d, mean_r))
|
| 200 |
+
return len(runs)
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
def main():
|
| 204 |
+
ap = argparse.ArgumentParser()
|
| 205 |
+
ap.add_argument("files", nargs="+", help="video clips to scan")
|
| 206 |
+
ap.add_argument("--head-frames", type=int, default=22,
|
| 207 |
+
help="frames of pinned head to report separately; match "
|
| 208 |
+
"your context_length, or 0 to skip")
|
| 209 |
+
ap.add_argument("--rel", type=float, default=0.15,
|
| 210 |
+
help="fraction of the clip's median frame motion below "
|
| 211 |
+
"which a frame counts as still")
|
| 212 |
+
ap.add_argument("--abs-floor", type=float, default=0.15,
|
| 213 |
+
help="absolute motion floor on a 0-255 scale, for "
|
| 214 |
+
"duplicated frames in an already still shot")
|
| 215 |
+
ap.add_argument("--min-ms", type=float, default=400.0,
|
| 216 |
+
help="shortest stretch worth reporting")
|
| 217 |
+
args = ap.parse_args()
|
| 218 |
+
|
| 219 |
+
total = 0
|
| 220 |
+
for path in args.files:
|
| 221 |
+
total += report(path, args.head_frames, args.rel, args.abs_floor,
|
| 222 |
+
args.min_ms)
|
| 223 |
+
print()
|
| 224 |
+
print("%d freeze run(s) across %d clip(s)" % (total, len(args.files)))
|
| 225 |
+
if total:
|
| 226 |
+
print("A held framing needs a breath, a weight shift or an eyeline "
|
| 227 |
+
"change written into the prompt. The camera holds; the "
|
| 228 |
+
"performer should not.")
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def _self_test():
|
| 232 |
+
"""Synthetic checks for the detection maths, no video files needed."""
|
| 233 |
+
rng = np.random.default_rng(0)
|
| 234 |
+
fps = 24.0
|
| 235 |
+
|
| 236 |
+
# 120 frames of ordinary motion with a 48 frame (2s) freeze at 30
|
| 237 |
+
deltas = rng.uniform(1.8, 2.2, 120)
|
| 238 |
+
deltas[30:78] = rng.uniform(0.0, 0.02, 48)
|
| 239 |
+
runs = find_runs(deltas, fps, min_ms=400.0)
|
| 240 |
+
assert len(runs) == 1, runs
|
| 241 |
+
a, b, mean_d, mean_r = runs[0]
|
| 242 |
+
assert a == 30 and b == 79, (a, b) # frames 30..78 identical
|
| 243 |
+
assert abs((b - 1 - a) / fps - 2.0) < 1e-9 # which is exactly 2.0s
|
| 244 |
+
assert mean_r < 0.02, mean_r
|
| 245 |
+
|
| 246 |
+
# a short dip must not be reported at the default 400 ms
|
| 247 |
+
short = rng.uniform(1.8, 2.2, 120)
|
| 248 |
+
short[50:55] = 0.01
|
| 249 |
+
assert find_runs(short, fps, min_ms=400.0) == []
|
| 250 |
+
assert len(find_runs(short, fps, min_ms=100.0)) == 1
|
| 251 |
+
|
| 252 |
+
# a dim clip with small absolute motion but no freeze must stay clean,
|
| 253 |
+
# which is the case an absolute-only threshold would fail
|
| 254 |
+
dim = rng.uniform(0.30, 0.34, 120)
|
| 255 |
+
assert find_runs(dim, fps, min_ms=400.0) == [], "dim clip false positive"
|
| 256 |
+
|
| 257 |
+
# and a freeze inside that same dim clip must still be caught
|
| 258 |
+
dim2 = dim.copy()
|
| 259 |
+
dim2[60:100] = 0.001
|
| 260 |
+
assert len(find_runs(dim2, fps, min_ms=400.0)) == 1
|
| 261 |
+
|
| 262 |
+
# a wholly static clip reports one run rather than dividing by zero
|
| 263 |
+
assert len(find_runs(np.zeros(60), fps)) == 1
|
| 264 |
+
|
| 265 |
+
# thumbnailing must average codec noise away: two frames of the same
|
| 266 |
+
# picture plus independent noise read as far less motion than two
|
| 267 |
+
# genuinely different pictures
|
| 268 |
+
base = rng.integers(0, 255, (360, 640, 3)).astype(np.uint8)
|
| 269 |
+
noisy = np.clip(base.astype(int) +
|
| 270 |
+
rng.integers(-6, 7, base.shape), 0, 255).astype(np.uint8)
|
| 271 |
+
other = rng.integers(0, 255, (360, 640, 3)).astype(np.uint8)
|
| 272 |
+
still_d = float(np.abs(_to_thumb(base) - _to_thumb(noisy)).mean())
|
| 273 |
+
moved_d = float(np.abs(_to_thumb(base) - _to_thumb(other)).mean())
|
| 274 |
+
assert still_d < 0.5, still_d
|
| 275 |
+
assert moved_d > 10 * still_d, (still_d, moved_d)
|
| 276 |
+
|
| 277 |
+
print("freeze_detect self-test passed "
|
| 278 |
+
"(still %.3f vs moved %.3f after thumbnailing)" %
|
| 279 |
+
(still_d, moved_d))
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
if __name__ == "__main__":
|
| 283 |
+
if "--self-test" in sys.argv:
|
| 284 |
+
_self_test()
|
| 285 |
+
else:
|
| 286 |
+
main()
|
custom_nodes/ComfyUI-H3-Motion-Context-main/tests/level_step.py
ADDED
|
@@ -0,0 +1,334 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Level step probe: does the sound level jump at a join?
|
| 2 |
+
|
| 3 |
+
Companion to seam_probe.py. seam_probe answers "is clip B's audio a real
|
| 4 |
+
continuation of clip A's, or an imitation" by cross-correlation. This
|
| 5 |
+
answers a different and simpler question: "does the loudness or the room
|
| 6 |
+
tone change abruptly across the cut?" A chain can be perfectly phase
|
| 7 |
+
locked and still audibly step if one clip invents its own silence.
|
| 8 |
+
|
| 9 |
+
Usage:
|
| 10 |
+
python level_step.py clip01.flac clip02.flac clip03.flac ...
|
| 11 |
+
python level_step.py --joins 12.5,25.0 episode.wav
|
| 12 |
+
|
| 13 |
+
Give the DELIVERED clips in chain order (trimmed, the ones you actually
|
| 14 |
+
concatenate), or one already concatenated file plus the join times in
|
| 15 |
+
seconds. Either way each join is scored twice:
|
| 16 |
+
|
| 17 |
+
broadband RMS of the window either side. Catches gain jumps.
|
| 18 |
+
floor 10th percentile of short-window RMS, which tracks the
|
| 19 |
+
room tone under the content. Catches the failure where
|
| 20 |
+
each clip invents its own silence and the ambience
|
| 21 |
+
restarts at every cut, even though the music matches.
|
| 22 |
+
|
| 23 |
+
Both are reported as a normalised step in [0, 1]:
|
| 24 |
+
|
| 25 |
+
step = |a - b| / (a + b)
|
| 26 |
+
|
| 27 |
+
so 0.0 is identical level, 1.0 means one side is digitally silent. The
|
| 28 |
+
dB difference is printed alongside because it is easier to judge by ear.
|
| 29 |
+
|
| 30 |
+
Reading the result:
|
| 31 |
+
floor < 0.25 clean, the bed runs through the cut
|
| 32 |
+
floor 0.25 to 0.50 marginal, listen to it
|
| 33 |
+
floor > 0.50 audible step
|
| 34 |
+
|
| 35 |
+
Broadband is judged more loosely (ok below 0.40) because the content
|
| 36 |
+
either side of a real cut is different by definition, so some movement
|
| 37 |
+
there is the edit, not a fault. The floor is the one that should hold
|
| 38 |
+
still. If broadband is fine and floor is bad,
|
| 39 |
+
the ambience is resetting: wire the previous clip's
|
| 40 |
+
audio (context_latent, or context_audio + audio_vae)
|
| 41 |
+
into Motion Context so the next clip hears it.
|
| 42 |
+
|
| 43 |
+
One limitation worth knowing: the floor figure reads the quiet moments
|
| 44 |
+
between content, so it only means something when the content lets up.
|
| 45 |
+
Wall to wall music with no gaps has nothing underneath it to measure and
|
| 46 |
+
the floor number collapses onto the broadband one. Dialogue, ambience
|
| 47 |
+
and anything with rests are where it earns its keep.
|
| 48 |
+
|
| 49 |
+
Sample rates are printed per file and any mismatch is called out loudly.
|
| 50 |
+
A stream-copy concat cannot change rate mid stream, so mixing 32 kHz H3
|
| 51 |
+
output with a 48 kHz assumption decodes the tail to silence while every
|
| 52 |
+
duration assert stays green. H3 emits 32 kHz. Read it, never assume it.
|
| 53 |
+
|
| 54 |
+
Dependencies: numpy, plus ONE of torchaudio / soundfile / stdlib wave.
|
| 55 |
+
ComfyUI's embedded python has numpy and torchaudio; run it with that
|
| 56 |
+
interpreter if your system python lacks them.
|
| 57 |
+
"""
|
| 58 |
+
|
| 59 |
+
import argparse
|
| 60 |
+
import sys
|
| 61 |
+
|
| 62 |
+
import numpy as np
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def load_audio(path):
|
| 66 |
+
"""Return (mono float64 array, sample_rate). Same loader as seam_probe."""
|
| 67 |
+
try:
|
| 68 |
+
import torchaudio
|
| 69 |
+
wav, sr = torchaudio.load(path)
|
| 70 |
+
return wav.mean(dim=0).numpy().astype(np.float64), int(sr)
|
| 71 |
+
except Exception:
|
| 72 |
+
pass
|
| 73 |
+
try:
|
| 74 |
+
import soundfile as sf
|
| 75 |
+
data, sr = sf.read(path, always_2d=True)
|
| 76 |
+
return data.mean(axis=1).astype(np.float64), int(sr)
|
| 77 |
+
except Exception:
|
| 78 |
+
pass
|
| 79 |
+
import wave
|
| 80 |
+
with wave.open(path, "rb") as w:
|
| 81 |
+
sr = w.getframerate()
|
| 82 |
+
n = w.getnframes()
|
| 83 |
+
width = w.getsampwidth()
|
| 84 |
+
ch = w.getnchannels()
|
| 85 |
+
raw = w.readframes(n)
|
| 86 |
+
dtype = {1: np.uint8, 2: np.int16, 4: np.int32}.get(width)
|
| 87 |
+
if dtype is None:
|
| 88 |
+
raise RuntimeError("unsupported wav sample width %d" % width)
|
| 89 |
+
data = np.frombuffer(raw, dtype=dtype).astype(np.float64)
|
| 90 |
+
if width == 1:
|
| 91 |
+
data -= 128.0
|
| 92 |
+
scale = {1: 128.0, 2: 32768.0, 4: 2147483648.0}[width]
|
| 93 |
+
data = data.reshape(-1, ch).mean(axis=1) / scale
|
| 94 |
+
return data, sr
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def _pair_label(a_name, b_name):
|
| 98 |
+
import os
|
| 99 |
+
short = lambda p: os.path.splitext(os.path.basename(p))[0][-8:]
|
| 100 |
+
return "%s|%s" % (short(a_name), short(b_name))
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def rms(x):
|
| 104 |
+
if len(x) == 0:
|
| 105 |
+
return 0.0
|
| 106 |
+
return float(np.sqrt(np.mean(np.square(x))))
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def floor_level(x, sr, win_ms=20.0, pct=10.0):
|
| 110 |
+
"""Noise floor estimate: a low percentile of short-window RMS.
|
| 111 |
+
|
| 112 |
+
The quietest moments of a window are room tone plus whatever the
|
| 113 |
+
encoder left behind. Content (a note, a word) raises the mean but
|
| 114 |
+
barely moves the 10th percentile, so this isolates the bed that is
|
| 115 |
+
supposed to run continuously underneath the whole chain.
|
| 116 |
+
"""
|
| 117 |
+
win = max(1, int(round(win_ms / 1000.0 * sr)))
|
| 118 |
+
n = len(x) // win
|
| 119 |
+
if n < 3:
|
| 120 |
+
return rms(x)
|
| 121 |
+
trimmed = x[:n * win].reshape(n, win)
|
| 122 |
+
levels = np.sqrt(np.mean(np.square(trimmed), axis=1))
|
| 123 |
+
return float(np.percentile(levels, pct))
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def step_ratio(a, b):
|
| 127 |
+
"""Normalised difference in [0, 1]. 1.0 when one side is silent."""
|
| 128 |
+
denom = a + b
|
| 129 |
+
if denom <= 1e-12:
|
| 130 |
+
return 0.0
|
| 131 |
+
return abs(a - b) / denom
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def db_delta(a, b):
|
| 135 |
+
"""Level change from a to b in dB, clamped so digital silence on one
|
| 136 |
+
side prints as a readable -99 rather than a wall of digits."""
|
| 137 |
+
if a <= 1e-9 and b <= 1e-9:
|
| 138 |
+
return 0.0
|
| 139 |
+
lo = 1e-9
|
| 140 |
+
d = 20.0 * np.log10(max(b, lo) / max(a, lo))
|
| 141 |
+
return float(np.clip(d, -99.0, 99.0))
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
# broadband level legitimately moves across a real cut, because the
|
| 145 |
+
# content on either side is different by definition. The floor should
|
| 146 |
+
# not: the bed under the content is supposed to run continuously
|
| 147 |
+
# through the whole chain. So they get different thresholds.
|
| 148 |
+
BROADBAND_OK, BROADBAND_BAD = 0.40, 0.65
|
| 149 |
+
FLOOR_OK, FLOOR_BAD = 0.25, 0.50
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def verdict(step, ok=0.25, bad=0.50):
|
| 153 |
+
if step < ok:
|
| 154 |
+
return "ok"
|
| 155 |
+
if step < bad:
|
| 156 |
+
return "MARGINAL"
|
| 157 |
+
return "STEP"
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def score_join(tail, head, sr, label):
|
| 161 |
+
"""Score one join given the audio either side of it."""
|
| 162 |
+
b_tail, b_head = rms(tail), rms(head)
|
| 163 |
+
f_tail, f_head = floor_level(tail, sr), floor_level(head, sr)
|
| 164 |
+
b_step = step_ratio(b_tail, b_head)
|
| 165 |
+
f_step = step_ratio(f_tail, f_head)
|
| 166 |
+
print("%-20s broadband %.3f (%+6.1f dB) %-8s "
|
| 167 |
+
"floor %.3f (%+6.1f dB) %-8s" %
|
| 168 |
+
(label, b_step, db_delta(b_tail, b_head),
|
| 169 |
+
verdict(b_step, BROADBAND_OK, BROADBAND_BAD),
|
| 170 |
+
f_step, db_delta(f_tail, f_head),
|
| 171 |
+
verdict(f_step, FLOOR_OK, FLOOR_BAD)))
|
| 172 |
+
return b_step, f_step
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def from_clips(paths, win_ms, guard_ms):
|
| 176 |
+
rates, joins = [], []
|
| 177 |
+
prev = None
|
| 178 |
+
for p in paths:
|
| 179 |
+
data, sr = load_audio(p)
|
| 180 |
+
rates.append((p, sr, len(data) / sr))
|
| 181 |
+
if prev is not None:
|
| 182 |
+
joins.append((prev[0], data, sr, prev[1], p))
|
| 183 |
+
prev = (data, p)
|
| 184 |
+
|
| 185 |
+
print("clips:")
|
| 186 |
+
for p, sr, dur in rates:
|
| 187 |
+
print(" %-40s %6d Hz %7.3fs" % (p[-40:], sr, dur))
|
| 188 |
+
uniq = sorted(set(sr for _, sr, _ in rates))
|
| 189 |
+
if len(uniq) > 1:
|
| 190 |
+
print()
|
| 191 |
+
print(" !! SAMPLE RATE MISMATCH across the chain: %s" % uniq)
|
| 192 |
+
print(" A stream-copy concat cannot switch rate mid stream. "
|
| 193 |
+
"Resample to one rate before joining, or the tail decodes "
|
| 194 |
+
"to silence with every duration check still passing.")
|
| 195 |
+
print()
|
| 196 |
+
|
| 197 |
+
win = None
|
| 198 |
+
results = []
|
| 199 |
+
print("%-20s %-42s %s" % ("join", "broadband", "floor"))
|
| 200 |
+
for a_data, b_data, sr, a_name, b_name in joins:
|
| 201 |
+
win = int(round(win_ms / 1000.0 * sr))
|
| 202 |
+
guard = int(round(guard_ms / 1000.0 * sr))
|
| 203 |
+
tail = a_data[max(0, len(a_data) - guard - win):len(a_data) - guard]
|
| 204 |
+
head = b_data[guard:guard + win]
|
| 205 |
+
if len(tail) < win // 2 or len(head) < win // 2:
|
| 206 |
+
print("%-20s too short to score" % _pair_label(a_name, b_name))
|
| 207 |
+
continue
|
| 208 |
+
results.append(score_join(tail, head, sr,
|
| 209 |
+
_pair_label(a_name, b_name)))
|
| 210 |
+
return results
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def from_single(path, join_times, win_ms, guard_ms):
|
| 214 |
+
data, sr = load_audio(path)
|
| 215 |
+
print("file: %s %d Hz %.3fs" % (path, sr, len(data) / sr))
|
| 216 |
+
print()
|
| 217 |
+
win = int(round(win_ms / 1000.0 * sr))
|
| 218 |
+
guard = int(round(guard_ms / 1000.0 * sr))
|
| 219 |
+
results = []
|
| 220 |
+
print("%-20s %-42s %s" % ("join", "broadband", "floor"))
|
| 221 |
+
for t in join_times:
|
| 222 |
+
c = int(round(t * sr))
|
| 223 |
+
tail = data[max(0, c - guard - win):max(0, c - guard)]
|
| 224 |
+
head = data[min(len(data), c + guard):min(len(data), c + guard + win)]
|
| 225 |
+
if len(tail) < win // 2 or len(head) < win // 2:
|
| 226 |
+
print("%-20s outside the file" % ("%.2fs" % t))
|
| 227 |
+
continue
|
| 228 |
+
results.append(score_join(tail, head, sr, "%.2fs" % t))
|
| 229 |
+
return results
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def main():
|
| 233 |
+
ap = argparse.ArgumentParser()
|
| 234 |
+
ap.add_argument("files", nargs="+",
|
| 235 |
+
help="delivered clips in chain order, or one "
|
| 236 |
+
"concatenated file with --joins")
|
| 237 |
+
ap.add_argument("--joins", default=None,
|
| 238 |
+
help="comma separated join times in seconds; use with "
|
| 239 |
+
"a single concatenated file")
|
| 240 |
+
ap.add_argument("--win-ms", type=float, default=400.0,
|
| 241 |
+
help="measurement window either side of the join. Long "
|
| 242 |
+
"enough to span typical gaps in the content: a "
|
| 243 |
+
"window that lands entirely inside a musical rest "
|
| 244 |
+
"on one side and a downbeat on the other reports a "
|
| 245 |
+
"step that is really just the material")
|
| 246 |
+
ap.add_argument("--guard-ms", type=float, default=5.0,
|
| 247 |
+
help="skipped either side, so a single splice sample "
|
| 248 |
+
"cannot dominate the window")
|
| 249 |
+
args = ap.parse_args()
|
| 250 |
+
|
| 251 |
+
if args.joins:
|
| 252 |
+
if len(args.files) != 1:
|
| 253 |
+
sys.exit("--joins takes exactly one concatenated file")
|
| 254 |
+
times = [float(t) for t in args.joins.split(",") if t.strip()]
|
| 255 |
+
results = from_single(args.files[0], times, args.win_ms, args.guard_ms)
|
| 256 |
+
else:
|
| 257 |
+
if len(args.files) < 2:
|
| 258 |
+
sys.exit("give at least two clips, or one file with --joins")
|
| 259 |
+
results = from_clips(args.files, args.win_ms, args.guard_ms)
|
| 260 |
+
|
| 261 |
+
print()
|
| 262 |
+
if not results:
|
| 263 |
+
sys.exit("no joins scored")
|
| 264 |
+
b = np.array([r[0] for r in results])
|
| 265 |
+
f = np.array([r[1] for r in results])
|
| 266 |
+
print("summary over %d joins:" % len(results))
|
| 267 |
+
print(" broadband median %.3f worst %.3f passing (<%.2f) %d/%d" %
|
| 268 |
+
(np.median(b), b.max(), BROADBAND_OK,
|
| 269 |
+
int((b < BROADBAND_OK).sum()), len(b)))
|
| 270 |
+
print(" floor median %.3f worst %.3f passing (<%.2f) %d/%d" %
|
| 271 |
+
(np.median(f), f.max(), FLOOR_OK,
|
| 272 |
+
int((f < FLOOR_OK).sum()), len(f)))
|
| 273 |
+
if np.median(f) >= FLOOR_OK and np.median(b) < BROADBAND_OK:
|
| 274 |
+
print()
|
| 275 |
+
print(" -> AMBIENCE RESET: content levels match but the bed under "
|
| 276 |
+
"them does not. Each clip is inventing its own room tone. "
|
| 277 |
+
"Carry the previous clip's audio across the join "
|
| 278 |
+
"(context_latent, or context_audio + audio_vae).")
|
| 279 |
+
elif np.median(b) >= BROADBAND_BAD:
|
| 280 |
+
print()
|
| 281 |
+
print(" -> LEVEL STEP: check for a gain change between clips, or "
|
| 282 |
+
"a normalisation pass applied per clip instead of once at "
|
| 283 |
+
"the end.")
|
| 284 |
+
elif np.median(b) < BROADBAND_OK and np.median(f) < FLOOR_OK:
|
| 285 |
+
print()
|
| 286 |
+
print(" -> joins are level-continuous. If they still sound wrong, "
|
| 287 |
+
"run seam_probe.py: the problem is phase, not level.")
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
def _self_test():
|
| 291 |
+
"""Synthetic checks for the scoring maths, no audio files needed."""
|
| 292 |
+
sr = 32000
|
| 293 |
+
rng = np.random.default_rng(0)
|
| 294 |
+
t = np.arange(sr) / sr
|
| 295 |
+
# gated tone, not a continuous one: the floor estimator reads the
|
| 296 |
+
# quiet moments between content, so content that never stops has no
|
| 297 |
+
# floor underneath it to find (see the note in the module docstring)
|
| 298 |
+
gate = ((t * 4).astype(int) % 2 == 0).astype(np.float64)
|
| 299 |
+
tone = 0.3 * np.sin(2 * np.pi * 220 * t) * gate
|
| 300 |
+
bed = 0.01 * rng.standard_normal(sr)
|
| 301 |
+
|
| 302 |
+
assert abs(step_ratio(0.5, 0.5)) < 1e-12
|
| 303 |
+
assert abs(step_ratio(0.5, 0.0) - 1.0) < 1e-12
|
| 304 |
+
assert abs(step_ratio(0.0, 0.0)) < 1e-12
|
| 305 |
+
assert 0.0 < step_ratio(0.5, 0.4) < 0.2
|
| 306 |
+
|
| 307 |
+
# a floor estimate must track the bed, not the tone on top of it
|
| 308 |
+
with_bed = floor_level(tone + bed, sr)
|
| 309 |
+
without = floor_level(tone, sr)
|
| 310 |
+
assert with_bed > without * 2, (with_bed, without)
|
| 311 |
+
assert abs(with_bed - rms(bed)) < rms(bed), (with_bed, rms(bed))
|
| 312 |
+
|
| 313 |
+
# ambience reset: same content, one side has no bed under it
|
| 314 |
+
a_step = step_ratio(rms(tone + bed), rms(tone))
|
| 315 |
+
f_step = step_ratio(floor_level(tone + bed, sr), floor_level(tone, sr))
|
| 316 |
+
assert a_step < 0.25, a_step # broadband says fine
|
| 317 |
+
assert f_step > 0.50, f_step # floor catches it
|
| 318 |
+
assert verdict(a_step, BROADBAND_OK, BROADBAND_BAD) == "ok"
|
| 319 |
+
assert verdict(f_step, FLOOR_OK, FLOOR_BAD) == "STEP"
|
| 320 |
+
|
| 321 |
+
# a 6 dB gain jump must register broadband
|
| 322 |
+
assert step_ratio(rms(tone), rms(tone * 2)) > 0.3
|
| 323 |
+
assert abs(db_delta(rms(tone), rms(tone * 2)) - 6.02) < 0.05
|
| 324 |
+
assert db_delta(0.1, 0.0) == -99.0 and db_delta(0.0, 0.1) == 99.0
|
| 325 |
+
assert db_delta(0.0, 0.0) == 0.0
|
| 326 |
+
|
| 327 |
+
print("level_step self-test passed")
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
if __name__ == "__main__":
|
| 331 |
+
if "--self-test" in sys.argv:
|
| 332 |
+
_self_test()
|
| 333 |
+
else:
|
| 334 |
+
main()
|
custom_nodes/ComfyUI-H3-Motion-Context-main/tests/seam_probe.py
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Seam probe: is clip B's regenerated head audio a drifting copy of clip
|
| 2 |
+
A's tail, or unrelated noise?
|
| 3 |
+
|
| 4 |
+
Usage:
|
| 5 |
+
python seam_probe.py clip_A.flac clip_B_UNTRIMMED.flac [--frames 22] [--fps 24]
|
| 6 |
+
|
| 7 |
+
clip_B_UNTRIMMED must be B's audio BEFORE the trim node -- branch the audio
|
| 8 |
+
VAE decode output to a Save Audio node in parallel with the trim. clip_A is
|
| 9 |
+
the previous clip as delivered (tail-matched is fine; match_tail only
|
| 10 |
+
removes ~8ms and the probe aligns on content, not file length).
|
| 11 |
+
|
| 12 |
+
The pinned video covers B's frames 0..N-1, which the model generates as a
|
| 13 |
+
reconstruction of A's last N frames. This script slides short windows of
|
| 14 |
+
B's head over the corresponding region of A's tail and reports, per window:
|
| 15 |
+
|
| 16 |
+
t_B window position in B (seconds from B's start)
|
| 17 |
+
lag_ms how far B runs behind A (positive = B is LATE: its copy of
|
| 18 |
+
the bassline arrives after A's original did). Tracked with
|
| 19 |
+
period-ambiguity handling, so a repeating bassline does not
|
| 20 |
+
fake a drift by cycle-hopping.
|
| 21 |
+
corr normalised cross-correlation peak (1.0 = identical shape,
|
| 22 |
+
~0 = unrelated)
|
| 23 |
+
|
| 24 |
+
Reading the result:
|
| 25 |
+
corr high (>0.6) and lag ~0 throughout clean continuation; the
|
| 26 |
+
blip is something else
|
| 27 |
+
corr high, lag grows/shifts toward the seam PHASE DRIFT -- crossfade
|
| 28 |
+
at the join is the fix,
|
| 29 |
+
shorter video window
|
| 30 |
+
should shrink it
|
| 31 |
+
corr low from the start the ref path is not
|
| 32 |
+
continuing A's audio at
|
| 33 |
+
all; windowing will never
|
| 34 |
+
fix it (idea 4 territory)
|
| 35 |
+
|
| 36 |
+
One limitation to know before trusting a lag. Cross-correlation cannot
|
| 37 |
+
tell a true alignment from one a whole cycle away, so on strongly
|
| 38 |
+
periodic content a lag larger than half a period reads as the nearest
|
| 39 |
+
alias: 60 ms on material with an 18 ms period comes back as -12.7 ms, at
|
| 40 |
+
high correlation. Beat-driven music is exactly this case. Tracking keeps
|
| 41 |
+
a drift from cycle-hopping mid-span, and the first window takes the
|
| 42 |
+
global peak so a uniform whole-step offset is visible at all, but a large
|
| 43 |
+
uniform offset on periodic content is not something the method resolves.
|
| 44 |
+
Read a small lag as reliable and a large one as "at least this much,
|
| 45 |
+
possibly plus a whole number of cycles".
|
| 46 |
+
|
| 47 |
+
The node does the same measurement without the file round trip, and
|
| 48 |
+
without having to infer where the seam is: H3 Motion Context Seam Probe,
|
| 49 |
+
wired between the audio VAE decode and the trim node.
|
| 50 |
+
|
| 51 |
+
Dependencies: numpy, plus ONE of torchaudio / soundfile / stdlib wave
|
| 52 |
+
(16/32-bit PCM wav). ComfyUI's embedded python has numpy and torchaudio.
|
| 53 |
+
Run it with that interpreter if your system python lacks them, e.g.:
|
| 54 |
+
python_embeded\\python.exe seam_probe.py A.flac B_untrimmed.flac
|
| 55 |
+
"""
|
| 56 |
+
|
| 57 |
+
import argparse
|
| 58 |
+
import sys
|
| 59 |
+
|
| 60 |
+
import numpy as np
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def load_audio(path):
|
| 64 |
+
"""Return (mono float64 array, sample_rate)."""
|
| 65 |
+
try:
|
| 66 |
+
import torchaudio
|
| 67 |
+
wav, sr = torchaudio.load(path)
|
| 68 |
+
return wav.mean(dim=0).numpy().astype(np.float64), int(sr)
|
| 69 |
+
except Exception:
|
| 70 |
+
pass
|
| 71 |
+
try:
|
| 72 |
+
import soundfile as sf
|
| 73 |
+
data, sr = sf.read(path, always_2d=True)
|
| 74 |
+
return data.mean(axis=1).astype(np.float64), int(sr)
|
| 75 |
+
except Exception:
|
| 76 |
+
pass
|
| 77 |
+
import wave
|
| 78 |
+
with wave.open(path, "rb") as w:
|
| 79 |
+
sr = w.getframerate()
|
| 80 |
+
n = w.getnframes()
|
| 81 |
+
width = w.getsampwidth()
|
| 82 |
+
ch = w.getnchannels()
|
| 83 |
+
raw = w.readframes(n)
|
| 84 |
+
dtype = {1: np.uint8, 2: np.int16, 4: np.int32}.get(width)
|
| 85 |
+
if dtype is None:
|
| 86 |
+
raise RuntimeError("unsupported wav sample width %d" % width)
|
| 87 |
+
data = np.frombuffer(raw, dtype=dtype).astype(np.float64)
|
| 88 |
+
if width == 1:
|
| 89 |
+
data -= 128.0
|
| 90 |
+
data = data.reshape(-1, ch).mean(axis=1)
|
| 91 |
+
return data, sr
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def norm_xcorr(win, ref):
|
| 95 |
+
"""Slide `win` across `ref`; return (offsets, ncc) where offsets index
|
| 96 |
+
positions of the window's start within ref."""
|
| 97 |
+
win = win - win.mean()
|
| 98 |
+
ref = ref - ref.mean()
|
| 99 |
+
wn = np.sqrt((win ** 2).sum())
|
| 100 |
+
if wn < 1e-12:
|
| 101 |
+
return None, None
|
| 102 |
+
corr = np.correlate(ref, win, mode="valid")
|
| 103 |
+
csum = np.concatenate(([0.0], np.cumsum(ref ** 2)))
|
| 104 |
+
seg = csum[len(win):] - csum[:-len(win)]
|
| 105 |
+
denom = wn * np.sqrt(np.maximum(seg, 1e-12))
|
| 106 |
+
return np.arange(len(corr)), corr / denom
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def tracked_peak(ncc, expect_idx):
|
| 110 |
+
"""Pick this window's alignment with period-ambiguity handling.
|
| 111 |
+
|
| 112 |
+
Music is periodic -- a 100 Hz bassline repeats every 10 ms -- so the
|
| 113 |
+
global NCC argmax can hop a whole cycle between windows and fake a
|
| 114 |
+
drift. Instead: take every local maximum within 10% of the global
|
| 115 |
+
peak, and follow the one CLOSEST to the alignment predicted by the
|
| 116 |
+
previous window (starting from zero lag). Genuine drift moves far
|
| 117 |
+
less than a period per 25 ms hop, so continuity keeps the true peak
|
| 118 |
+
and discards cycle aliases.
|
| 119 |
+
"""
|
| 120 |
+
best = float(ncc.max())
|
| 121 |
+
i = np.arange(1, len(ncc) - 1)
|
| 122 |
+
peaks = i[(ncc[i] >= ncc[i - 1]) & (ncc[i] >= ncc[i + 1]) &
|
| 123 |
+
(ncc[i] >= 0.9 * best)]
|
| 124 |
+
if len(peaks) == 0:
|
| 125 |
+
peaks = np.array([int(np.argmax(ncc))])
|
| 126 |
+
pick = int(peaks[np.argmin(np.abs(peaks - expect_idx))])
|
| 127 |
+
return pick, float(ncc[pick])
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def main():
|
| 131 |
+
ap = argparse.ArgumentParser()
|
| 132 |
+
ap.add_argument("clip_a")
|
| 133 |
+
ap.add_argument("clip_b_untrimmed")
|
| 134 |
+
ap.add_argument("--frames", type=int, default=22,
|
| 135 |
+
help="pinned video frames (the span the model regenerates)")
|
| 136 |
+
ap.add_argument("--fps", type=float, default=24.0)
|
| 137 |
+
ap.add_argument("--win-ms", type=float, default=50.0,
|
| 138 |
+
help="analysis window length")
|
| 139 |
+
ap.add_argument("--hop-ms", type=float, default=25.0,
|
| 140 |
+
help="hop between windows (25ms = one audio latent step)")
|
| 141 |
+
ap.add_argument("--search-ms", type=float, default=40.0,
|
| 142 |
+
help="max lag searched either side")
|
| 143 |
+
args = ap.parse_args()
|
| 144 |
+
|
| 145 |
+
a, sr_a = load_audio(args.clip_a)
|
| 146 |
+
b, sr_b = load_audio(args.clip_b_untrimmed)
|
| 147 |
+
if sr_a != sr_b:
|
| 148 |
+
sys.exit("sample rates differ (%d vs %d); resample first" % (sr_a, sr_b))
|
| 149 |
+
sr = sr_a
|
| 150 |
+
|
| 151 |
+
span_s = args.frames / args.fps
|
| 152 |
+
span = int(round(span_s * sr))
|
| 153 |
+
if len(b) < span:
|
| 154 |
+
sys.exit("clip B shorter than the pinned span -- is this the "
|
| 155 |
+
"UNTRIMMED audio?")
|
| 156 |
+
if len(a) < span:
|
| 157 |
+
sys.exit("clip A shorter than the pinned span")
|
| 158 |
+
|
| 159 |
+
# B[0:span] should reconstruct A[-span:]
|
| 160 |
+
a_tail_start = len(a) - span
|
| 161 |
+
win = int(round(args.win_ms / 1000.0 * sr))
|
| 162 |
+
hop = int(round(args.hop_ms / 1000.0 * sr))
|
| 163 |
+
search = int(round(args.search_ms / 1000.0 * sr))
|
| 164 |
+
|
| 165 |
+
print("clip A: %.3fs clip B (untrimmed): %.3fs sr %d" %
|
| 166 |
+
(len(a) / sr, len(b) / sr, sr))
|
| 167 |
+
print("pinned span: %d frames = %.4fs = %d samples" %
|
| 168 |
+
(args.frames, span_s, span))
|
| 169 |
+
print("window %.0fms, hop %.0fms, search +/-%.0fms" %
|
| 170 |
+
(args.win_ms, args.hop_ms, args.search_ms))
|
| 171 |
+
print()
|
| 172 |
+
print("%8s %9s %6s region" % ("t_B (s)", "lag (ms)", "corr"))
|
| 173 |
+
|
| 174 |
+
lags, corrs = [], []
|
| 175 |
+
t = 0
|
| 176 |
+
prev_lag = None # samples; seeded from the first window's global peak
|
| 177 |
+
# continue a little past the seam to see whether alignment against
|
| 178 |
+
# A's (now nonexistent) continuation collapses, as it should
|
| 179 |
+
while t + win <= span + int(0.2 * sr) and t + win <= len(b):
|
| 180 |
+
centre_in_a = a_tail_start + t
|
| 181 |
+
lo = max(0, centre_in_a - search)
|
| 182 |
+
hi = min(len(a), centre_in_a + win + search)
|
| 183 |
+
ref = a[lo:hi]
|
| 184 |
+
if len(ref) <= win:
|
| 185 |
+
break
|
| 186 |
+
offsets, ncc = norm_xcorr(b[t:t + win], ref)
|
| 187 |
+
if ncc is None:
|
| 188 |
+
t += hop
|
| 189 |
+
continue
|
| 190 |
+
# index in the ncc curve where lag == prev_lag; sign convention:
|
| 191 |
+
# positive lag = B is LATE (its content matches an EARLIER part
|
| 192 |
+
# of A than the nominal position), so match offset = centre - lag
|
| 193 |
+
if prev_lag is None:
|
| 194 |
+
# Nothing to track from yet, so take the global peak. Seeding
|
| 195 |
+
# at zero instead assumes the first window is already
|
| 196 |
+
# aligned, and when the whole join is offset by a step that
|
| 197 |
+
# assumption makes the tracker pick the nearest cycle alias:
|
| 198 |
+
# a uniform 25 ms lag on 55 Hz-fundamental content reads as
|
| 199 |
+
# 6.8 ms. That is the failure this probe exists to find, so
|
| 200 |
+
# it must not be the one case it cannot see.
|
| 201 |
+
pick = int(np.argmax(ncc))
|
| 202 |
+
corr = float(ncc[pick])
|
| 203 |
+
else:
|
| 204 |
+
expect_idx = (centre_in_a - prev_lag) - lo
|
| 205 |
+
pick, corr = tracked_peak(ncc, expect_idx)
|
| 206 |
+
lag = centre_in_a - (lo + pick)
|
| 207 |
+
in_span = t + win <= span
|
| 208 |
+
tag = "reconstruction" if in_span else "past seam"
|
| 209 |
+
print("%8.3f %9.2f %6.3f %s" %
|
| 210 |
+
(t / sr, lag / sr * 1000.0, corr, tag))
|
| 211 |
+
if in_span:
|
| 212 |
+
lags.append(lag / sr * 1000.0)
|
| 213 |
+
corrs.append(corr)
|
| 214 |
+
if corr > 0.6:
|
| 215 |
+
prev_lag = lag # only track when the match is credible
|
| 216 |
+
t += hop
|
| 217 |
+
|
| 218 |
+
print()
|
| 219 |
+
if not corrs:
|
| 220 |
+
sys.exit("no windows analysed")
|
| 221 |
+
corrs = np.array(corrs)
|
| 222 |
+
lags = np.array(lags)
|
| 223 |
+
strong = corrs > 0.6
|
| 224 |
+
at_edge = np.abs(lags) >= (args.search_ms - 1.0)
|
| 225 |
+
print("summary over the reconstruction span:")
|
| 226 |
+
print(" mean corr %.3f windows with corr>0.6: %d/%d "
|
| 227 |
+
"lags pinned at search edge: %d" %
|
| 228 |
+
(corrs.mean(), int(strong.sum()), len(corrs), int(at_edge.sum())))
|
| 229 |
+
usable = strong & ~at_edge
|
| 230 |
+
if usable.sum() >= 5 and corrs.mean() > 0.55:
|
| 231 |
+
# fit lag = a*t + b over the credible windows; drift is a coherent
|
| 232 |
+
# trend, so demand small residuals before calling it that
|
| 233 |
+
idx = np.nonzero(usable)[0].astype(np.float64)
|
| 234 |
+
sl = lags[usable]
|
| 235 |
+
a_fit, b_fit = np.polyfit(idx, sl, 1)
|
| 236 |
+
resid = sl - (a_fit * idx + b_fit)
|
| 237 |
+
rms = float(np.sqrt((resid ** 2).mean()))
|
| 238 |
+
total = a_fit * (len(corrs) - 1)
|
| 239 |
+
print(" lag trend: %.2fms across the span, residual rms %.2fms" %
|
| 240 |
+
(total, rms))
|
| 241 |
+
if rms < 2.0 and corrs.mean() > 0.75 and abs(total) > 2.0:
|
| 242 |
+
print(" -> DRIFT: B tracks A with a coherent %.1fms slide. A "
|
| 243 |
+
"~40ms crossfade at the join should mask it; a shorter "
|
| 244 |
+
"video window should shrink it." % total)
|
| 245 |
+
elif rms < 2.0 and corrs.mean() > 0.75:
|
| 246 |
+
print(" -> B tracks A with stable phase. The blip is not "
|
| 247 |
+
"accumulated drift; look at the first generated steps "
|
| 248 |
+
"past the seam instead.")
|
| 249 |
+
else:
|
| 250 |
+
print(" -> PARTIAL RESEMBLANCE: B resembles A's tail but does "
|
| 251 |
+
"not phase-lock to it (mean corr %.2f, lag rms %.1fms). "
|
| 252 |
+
"Imitation, not continuation -- idea 4 territory."
|
| 253 |
+
% (corrs.mean(), rms))
|
| 254 |
+
else:
|
| 255 |
+
print(" -> LOW/INCOHERENT CORRELATION: the regenerated head does "
|
| 256 |
+
"not phase-track A's tail. The ref path is imitating, not "
|
| 257 |
+
"continuing; no window length will fix the seam. Idea 4 "
|
| 258 |
+
"(audio on the keyframe path) is the direction.")
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
if __name__ == "__main__":
|
| 262 |
+
main()
|
custom_nodes/ComfyUI-H3-Motion-Context-main/web/h3_motion_context.js
ADDED
|
@@ -0,0 +1,389 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { app } from "../../scripts/app.js";
|
| 2 |
+
import { api } from "../../scripts/api.js";
|
| 3 |
+
|
| 4 |
+
const LOAD = "MiniMaxH3MotionContextLoadLatent";
|
| 5 |
+
const SAVE = "MiniMaxH3MotionContextSaveLatent";
|
| 6 |
+
const CHAIN = "MiniMaxH3MotionContextChain";
|
| 7 |
+
const MAX = 9999;
|
| 8 |
+
|
| 9 |
+
const CSS = `
|
| 10 |
+
.h3mc-chain{display:flex;flex-direction:column;gap:6px;color:#ddd;font:12px/1.3 sans-serif;width:100%;box-sizing:border-box;padding:2px 0 4px;}
|
| 11 |
+
.h3mc-chain-row{display:flex;gap:6px;}
|
| 12 |
+
.h3mc-chain-row button{flex:1;cursor:pointer;border:1px solid #555;background:#2a2a2a;color:#ddd;border-radius:4px;padding:6px 4px;font:12px sans-serif;}
|
| 13 |
+
.h3mc-chain-row button:hover{background:#3a3a3a;}
|
| 14 |
+
.h3mc-chain-row button:disabled{opacity:.45;cursor:default;}
|
| 15 |
+
.h3mc-chain-row button.on{border-color:#6f8bbd;background:#1f2a3a;color:#c5d4ee;}
|
| 16 |
+
.h3mc-chain-meta{opacity:.75;font-size:11px;min-height:14px;}
|
| 17 |
+
`;
|
| 18 |
+
|
| 19 |
+
let cssOnce = false;
|
| 20 |
+
function injectCss() {
|
| 21 |
+
if (cssOnce) return;
|
| 22 |
+
cssOnce = true;
|
| 23 |
+
const s = document.createElement("style");
|
| 24 |
+
s.textContent = CSS;
|
| 25 |
+
document.head.appendChild(s);
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
function swallow(el) {
|
| 29 |
+
for (const type of ["pointerdown", "mousedown", "pointerup", "mouseup",
|
| 30 |
+
"click", "dblclick", "contextmenu"]) {
|
| 31 |
+
el.addEventListener(type, (e) => e.stopPropagation());
|
| 32 |
+
}
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
function graphNodes(graph) {
|
| 36 |
+
return graph?._nodes || graph?.nodes || [];
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
function groupList(graph) {
|
| 40 |
+
return graph?._groups || graph?.groups || [];
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
function groupMembers(group) {
|
| 44 |
+
if (typeof group.recomputeInsideNodes === "function") {
|
| 45 |
+
try { group.recomputeInsideNodes(); } catch (e) { /* older litegraph */ }
|
| 46 |
+
}
|
| 47 |
+
if (Array.isArray(group._nodes) && group._nodes.length) return group._nodes;
|
| 48 |
+
if (Array.isArray(group.nodes) && group.nodes.length) return group.nodes;
|
| 49 |
+
const kids = group._children || group.children;
|
| 50 |
+
if (kids) return Array.from(kids);
|
| 51 |
+
return [];
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
function inGroup(group, node) {
|
| 55 |
+
const members = groupMembers(group);
|
| 56 |
+
if (members.includes(node) || members.some((n) => n.id === node.id)) return true;
|
| 57 |
+
const b = group._bounding || group.bounding;
|
| 58 |
+
if (!b || b.length < 4 || !node.pos) return false;
|
| 59 |
+
const x = node.pos[0] + (node.size?.[0] || 0) / 2;
|
| 60 |
+
const y = node.pos[1] + (node.size?.[1] || 0) / 2;
|
| 61 |
+
return x >= b[0] && x <= b[0] + b[2] && y >= b[1] && y <= b[1] + b[3];
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
function findPair(ctrl) {
|
| 65 |
+
const graph = ctrl.graph || app.graph;
|
| 66 |
+
const nodes = graphNodes(graph);
|
| 67 |
+
const loads = nodes.filter((n) => n.comfyClass === LOAD);
|
| 68 |
+
const saves = nodes.filter((n) => n.comfyClass === SAVE);
|
| 69 |
+
for (const g of groupList(graph)) {
|
| 70 |
+
if (!inGroup(g, ctrl)) continue;
|
| 71 |
+
const members = groupMembers(g);
|
| 72 |
+
const pool = members.length ? members : nodes.filter((n) => inGroup(g, n));
|
| 73 |
+
const load = pool.find((n) => n.comfyClass === LOAD);
|
| 74 |
+
const save = pool.find((n) => n.comfyClass === SAVE);
|
| 75 |
+
if (load && save) return { load, save };
|
| 76 |
+
}
|
| 77 |
+
if (loads.length === 1 && saves.length === 1) return { load: loads[0], save: saves[0] };
|
| 78 |
+
return null;
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
function clipWidget(node) {
|
| 82 |
+
return node?.widgets?.find((w) => w.name === "clip_index");
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
function readClip(node) {
|
| 86 |
+
return (clipWidget(node)?.value | 0) || 0;
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
function widgetValue(node, name) {
|
| 90 |
+
return node?.widgets?.find((w) => w.name === name)?.value;
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
function loadPath(pair) {
|
| 94 |
+
const p = widgetValue(pair.load, "latent_path");
|
| 95 |
+
return (p == null || p === "") ? "h3_context" : p;
|
| 96 |
+
}
|
| 97 |
+
|
| 98 |
+
async function firstClipExists(pair) {
|
| 99 |
+
try {
|
| 100 |
+
const r = await api.fetchApi("/h3_motion_context/slot_exists", {
|
| 101 |
+
method: "POST",
|
| 102 |
+
headers: { "Content-Type": "application/json" },
|
| 103 |
+
body: JSON.stringify({ latent_path: loadPath(pair), clip_index: 1 }),
|
| 104 |
+
});
|
| 105 |
+
if (!r.ok) return false;
|
| 106 |
+
const j = await r.json();
|
| 107 |
+
return !!j.exists;
|
| 108 |
+
} catch (e) {
|
| 109 |
+
return false;
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
|
| 113 |
+
async function clearLatents(pair) {
|
| 114 |
+
try {
|
| 115 |
+
const r = await api.fetchApi("/h3_motion_context/clear_latents", {
|
| 116 |
+
method: "POST",
|
| 117 |
+
headers: { "Content-Type": "application/json" },
|
| 118 |
+
body: JSON.stringify({ latent_path: loadPath(pair) }),
|
| 119 |
+
});
|
| 120 |
+
if (!r.ok) return 0;
|
| 121 |
+
const j = await r.json();
|
| 122 |
+
return j.removed | 0;
|
| 123 |
+
} catch (e) {
|
| 124 |
+
return 0;
|
| 125 |
+
}
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
function writeClip(node, value) {
|
| 129 |
+
const w = clipWidget(node);
|
| 130 |
+
if (!w) return false;
|
| 131 |
+
const next = Math.max(0, Math.min(MAX, value | 0));
|
| 132 |
+
w.value = next;
|
| 133 |
+
return true;
|
| 134 |
+
}
|
| 135 |
+
|
| 136 |
+
function coerceSegments(node) {
|
| 137 |
+
const w = node?.widgets?.find((x) => x.name === "segments");
|
| 138 |
+
if (!w) return 0;
|
| 139 |
+
const n = parseInt(w.value, 10);
|
| 140 |
+
const v = Number.isFinite(n) && n >= 0 ? n : 0;
|
| 141 |
+
w.value = v;
|
| 142 |
+
return v;
|
| 143 |
+
}
|
| 144 |
+
|
| 145 |
+
function wireSegments(node) {
|
| 146 |
+
const w = node?.widgets?.find((x) => x.name === "segments");
|
| 147 |
+
if (!w || w._h3mcSeg) return;
|
| 148 |
+
w._h3mcSeg = true;
|
| 149 |
+
const prev = w.serializeValue?.bind(w);
|
| 150 |
+
w.serializeValue = async function (n, i) {
|
| 151 |
+
if (prev) {
|
| 152 |
+
try { await prev(n, i); } catch (e) { /* keep coercing */ }
|
| 153 |
+
}
|
| 154 |
+
return coerceSegments(node);
|
| 155 |
+
};
|
| 156 |
+
const prevCb = w.callback;
|
| 157 |
+
w.callback = function () {
|
| 158 |
+
coerceSegments(node);
|
| 159 |
+
return prevCb?.apply(this, arguments);
|
| 160 |
+
};
|
| 161 |
+
coerceSegments(node);
|
| 162 |
+
}
|
| 163 |
+
|
| 164 |
+
function readSegments(ctrl) {
|
| 165 |
+
return coerceSegments(ctrl);
|
| 166 |
+
}
|
| 167 |
+
|
| 168 |
+
function paint(ctrl) {
|
| 169 |
+
const meta = ctrl._h3mc?.meta;
|
| 170 |
+
if (!meta) return;
|
| 171 |
+
const pair = findPair(ctrl);
|
| 172 |
+
if (!pair) {
|
| 173 |
+
meta.textContent = "Load, Save, and Chain must share one canvas group.";
|
| 174 |
+
return;
|
| 175 |
+
}
|
| 176 |
+
const a = readClip(pair.load);
|
| 177 |
+
const b = readClip(pair.save);
|
| 178 |
+
const left = ctrl._h3mc.remaining | 0;
|
| 179 |
+
if (ctrl._h3mc.chaining) {
|
| 180 |
+
meta.textContent = left
|
| 181 |
+
? `Chaining Load ${a} / Save ${b} · ${left} left`
|
| 182 |
+
: `Chaining Load ${a} / Save ${b}`;
|
| 183 |
+
} else {
|
| 184 |
+
meta.textContent = `Load ${a} / Save ${b}`;
|
| 185 |
+
}
|
| 186 |
+
const chainBtn = ctrl._h3mc.chainBtn;
|
| 187 |
+
if (chainBtn) {
|
| 188 |
+
chainBtn.textContent = ctrl._h3mc.chaining ? "Stop" : "Chain";
|
| 189 |
+
chainBtn.classList.toggle("on", !!ctrl._h3mc.chaining);
|
| 190 |
+
}
|
| 191 |
+
}
|
| 192 |
+
|
| 193 |
+
let live = null;
|
| 194 |
+
|
| 195 |
+
function stopChain(ctrl) {
|
| 196 |
+
if (ctrl) {
|
| 197 |
+
ctrl._h3mc.chaining = false;
|
| 198 |
+
ctrl._h3mc.awaiting = false;
|
| 199 |
+
ctrl._h3mc.remaining = 0;
|
| 200 |
+
paint(ctrl);
|
| 201 |
+
}
|
| 202 |
+
if (live === ctrl) live = null;
|
| 203 |
+
}
|
| 204 |
+
|
| 205 |
+
function advance(ctrl) {
|
| 206 |
+
const pair = findPair(ctrl);
|
| 207 |
+
if (!pair) return false;
|
| 208 |
+
const load = readClip(pair.load) + 1;
|
| 209 |
+
let save = readClip(pair.save) + 1;
|
| 210 |
+
if (save < 1) save = 1;
|
| 211 |
+
writeClip(pair.load, Math.min(load, MAX));
|
| 212 |
+
writeClip(pair.save, Math.min(save, MAX));
|
| 213 |
+
app.graph?.setDirtyCanvas?.(true, true);
|
| 214 |
+
paint(ctrl);
|
| 215 |
+
return true;
|
| 216 |
+
}
|
| 217 |
+
|
| 218 |
+
function resetFirst(ctrl) {
|
| 219 |
+
const pair = findPair(ctrl);
|
| 220 |
+
if (!pair) return false;
|
| 221 |
+
writeClip(pair.load, 0);
|
| 222 |
+
writeClip(pair.save, 1);
|
| 223 |
+
app.graph?.setDirtyCanvas?.(true, true);
|
| 224 |
+
paint(ctrl);
|
| 225 |
+
return true;
|
| 226 |
+
}
|
| 227 |
+
|
| 228 |
+
async function startChain(ctrl) {
|
| 229 |
+
const pair = findPair(ctrl);
|
| 230 |
+
if (!pair) return false;
|
| 231 |
+
const load = readClip(pair.load);
|
| 232 |
+
const save = readClip(pair.save);
|
| 233 |
+
const atFirst = load === 0 && save <= 1;
|
| 234 |
+
if (atFirst && !(await firstClipExists(pair))) {
|
| 235 |
+
await queueOnce(ctrl);
|
| 236 |
+
return true;
|
| 237 |
+
}
|
| 238 |
+
if (!advance(ctrl)) return false;
|
| 239 |
+
await queueOnce(ctrl);
|
| 240 |
+
return true;
|
| 241 |
+
}
|
| 242 |
+
|
| 243 |
+
async function queueOnce(ctrl) {
|
| 244 |
+
const pair = findPair(ctrl);
|
| 245 |
+
if (!pair) return;
|
| 246 |
+
ctrl._h3mc.awaiting = true;
|
| 247 |
+
live = ctrl;
|
| 248 |
+
await app.queuePrompt(0, 1);
|
| 249 |
+
}
|
| 250 |
+
|
| 251 |
+
function onPromptDone(ok) {
|
| 252 |
+
const ctrl = live;
|
| 253 |
+
if (!ctrl?._h3mc?.awaiting) return;
|
| 254 |
+
ctrl._h3mc.awaiting = false;
|
| 255 |
+
if (!ok) {
|
| 256 |
+
stopChain(ctrl);
|
| 257 |
+
return;
|
| 258 |
+
}
|
| 259 |
+
if (!ctrl._h3mc.chaining) {
|
| 260 |
+
live = null;
|
| 261 |
+
paint(ctrl);
|
| 262 |
+
return;
|
| 263 |
+
}
|
| 264 |
+
const left = ctrl._h3mc.remaining | 0;
|
| 265 |
+
if (left > 0) {
|
| 266 |
+
ctrl._h3mc.remaining = left - 1;
|
| 267 |
+
if (ctrl._h3mc.remaining === 0) {
|
| 268 |
+
ctrl._h3mc.chaining = false;
|
| 269 |
+
live = null;
|
| 270 |
+
paint(ctrl);
|
| 271 |
+
return;
|
| 272 |
+
}
|
| 273 |
+
}
|
| 274 |
+
if (!advance(ctrl)) {
|
| 275 |
+
stopChain(ctrl);
|
| 276 |
+
return;
|
| 277 |
+
}
|
| 278 |
+
queueOnce(ctrl);
|
| 279 |
+
}
|
| 280 |
+
|
| 281 |
+
api.addEventListener("execution_success", () => onPromptDone(true));
|
| 282 |
+
api.addEventListener("execution_error", () => onPromptDone(false));
|
| 283 |
+
api.addEventListener("execution_interrupted", () => onPromptDone(false));
|
| 284 |
+
|
| 285 |
+
app.registerExtension({
|
| 286 |
+
name: "h3_motion_context.chain",
|
| 287 |
+
async beforeRegisterNodeDef(nodeType, nodeData) {
|
| 288 |
+
if (nodeData.name !== CHAIN) return;
|
| 289 |
+
const onNodeCreated = nodeType.prototype.onNodeCreated;
|
| 290 |
+
nodeType.prototype.onNodeCreated = function () {
|
| 291 |
+
const r = onNodeCreated?.apply(this, arguments);
|
| 292 |
+
wireSegments(this);
|
| 293 |
+
injectCss();
|
| 294 |
+
const root = document.createElement("div");
|
| 295 |
+
root.className = "h3mc-chain";
|
| 296 |
+
const row = document.createElement("div");
|
| 297 |
+
row.className = "h3mc-chain-row";
|
| 298 |
+
const row2 = document.createElement("div");
|
| 299 |
+
row2.className = "h3mc-chain-row";
|
| 300 |
+
const row3 = document.createElement("div");
|
| 301 |
+
row3.className = "h3mc-chain-row";
|
| 302 |
+
const approve = document.createElement("button");
|
| 303 |
+
approve.textContent = "Approve";
|
| 304 |
+
approve.title = "Advance Load/Save, then run the next clip.";
|
| 305 |
+
const reroll = document.createElement("button");
|
| 306 |
+
reroll.textContent = "Run/Re-roll";
|
| 307 |
+
reroll.title = "Queue at the current Load/Save indices. Use this instead of ComfyUI's Run button.";
|
| 308 |
+
const chainBtn = document.createElement("button");
|
| 309 |
+
chainBtn.textContent = "Chain";
|
| 310 |
+
chainBtn.title = "Approve on a loop. segments > 0 stops after that many clips; 0 runs until Stop.";
|
| 311 |
+
const resetBtn = document.createElement("button");
|
| 312 |
+
resetBtn.textContent = "Reset";
|
| 313 |
+
resetBtn.title = "Set Load 0 / Save 1. Does not queue or delete files.";
|
| 314 |
+
const clearBtn = document.createElement("button");
|
| 315 |
+
clearBtn.textContent = "Clear latents";
|
| 316 |
+
clearBtn.title = "Delete numbered chain slots (clip_00001.safetensors and so on). Custom filenames are left alone. Does not change indices or queue.";
|
| 317 |
+
row.append(approve, reroll);
|
| 318 |
+
row2.append(chainBtn, resetBtn);
|
| 319 |
+
row3.append(clearBtn);
|
| 320 |
+
const meta = document.createElement("div");
|
| 321 |
+
meta.className = "h3mc-chain-meta";
|
| 322 |
+
root.append(row, row2, row3, meta);
|
| 323 |
+
swallow(root);
|
| 324 |
+
this.addDOMWidget("h3mc_chain", "CHAIN", root, { serialize: false });
|
| 325 |
+
this._h3mc = { chaining: false, awaiting: false, remaining: 0, meta, chainBtn };
|
| 326 |
+
approve.onclick = async (e) => {
|
| 327 |
+
e.stopPropagation();
|
| 328 |
+
if (this._h3mc.awaiting) return;
|
| 329 |
+
stopChain(this);
|
| 330 |
+
if (!advance(this)) return;
|
| 331 |
+
await queueOnce(this);
|
| 332 |
+
};
|
| 333 |
+
reroll.onclick = async (e) => {
|
| 334 |
+
e.stopPropagation();
|
| 335 |
+
if (this._h3mc.awaiting) return;
|
| 336 |
+
stopChain(this);
|
| 337 |
+
await queueOnce(this);
|
| 338 |
+
};
|
| 339 |
+
chainBtn.onclick = async (e) => {
|
| 340 |
+
e.stopPropagation();
|
| 341 |
+
if (this._h3mc.chaining) {
|
| 342 |
+
stopChain(this);
|
| 343 |
+
return;
|
| 344 |
+
}
|
| 345 |
+
if (this._h3mc.awaiting) return;
|
| 346 |
+
if (!findPair(this)) {
|
| 347 |
+
paint(this);
|
| 348 |
+
return;
|
| 349 |
+
}
|
| 350 |
+
this._h3mc.chaining = true;
|
| 351 |
+
this._h3mc.remaining = readSegments(this);
|
| 352 |
+
paint(this);
|
| 353 |
+
if (!await startChain(this)) stopChain(this);
|
| 354 |
+
};
|
| 355 |
+
resetBtn.onclick = (e) => {
|
| 356 |
+
e.stopPropagation();
|
| 357 |
+
if (this._h3mc.awaiting) return;
|
| 358 |
+
stopChain(this);
|
| 359 |
+
resetFirst(this);
|
| 360 |
+
};
|
| 361 |
+
clearBtn.onclick = async (e) => {
|
| 362 |
+
e.stopPropagation();
|
| 363 |
+
if (this._h3mc.awaiting || this._h3mc.chaining) return;
|
| 364 |
+
const pair = findPair(this);
|
| 365 |
+
if (!pair) {
|
| 366 |
+
paint(this);
|
| 367 |
+
return;
|
| 368 |
+
}
|
| 369 |
+
const n = await clearLatents(pair);
|
| 370 |
+
paint(this);
|
| 371 |
+
if (this._h3mc?.meta) {
|
| 372 |
+
this._h3mc.meta.textContent = n
|
| 373 |
+
? `Removed ${n} numbered slot${n === 1 ? "" : "s"}. Custom names kept.`
|
| 374 |
+
: "No numbered chain slots to remove.";
|
| 375 |
+
}
|
| 376 |
+
};
|
| 377 |
+
paint(this);
|
| 378 |
+
this.setSize?.([270, 168]);
|
| 379 |
+
return r;
|
| 380 |
+
};
|
| 381 |
+
const onConfigure = nodeType.prototype.onConfigure;
|
| 382 |
+
nodeType.prototype.onConfigure = function () {
|
| 383 |
+
const r = onConfigure?.apply(this, arguments);
|
| 384 |
+
wireSegments(this);
|
| 385 |
+
coerceSegments(this);
|
| 386 |
+
return r;
|
| 387 |
+
};
|
| 388 |
+
},
|
| 389 |
+
});
|