File size: 8,705 Bytes
60b1546
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
"""H3 Steady Lab (Noah Oct 28): the stabilizer from H3 Steady with exposed controls for manual experimentation.

Same calculation as comfyui-h3-steady: for each second of new image tokens, mean m along direction w, m < b → + strength·(b−m)·w.
With the default settings (steps 1+2, blocks 42–48, threshold 90 %), results are bit-identical to H3 Steady.
Profile profil_lab.safetensors (film/forschung_zeit/steady_lab_profil.py): w [step 0–2, block 0–49, 5376],
b [quantile, step, block] for 50/75/90/95/100 % of the 28 clean segments. Affects image tokens only.
"""
import logging
import os

import torch
from safetensors.torch import load_file

from comfy_api.latest import ComfyExtension, io

_PROFIL = os.path.join(os.path.dirname(os.path.abspath(__file__)), "profil_lab.safetensors")
_FPS = 24
_SCHWELLEN = ["50 % (mild)", "75 %", "90 % (Standard)", "95 %", "100 % (strict)"]


def _frame(k):
    b, r = divmod(k, 5)
    return b * 17 + (0 if r == 0 else 1 + 4 * (r - 1))


def _sekunden(layout, offset, sek_von, sek_bis):
    """[(second, start row, end row)] of new image tokens, only seconds in the window [sek_von, sek_bis)."""
    _, latent_t, lat_h, lat_w, _ = layout.signature
    rows = (lat_h // 2) * (lat_w // 2)
    va, vb, _ = next(s for s in layout.segments if s[2] == "video")
    spans = {}
    for k in range(offset, latent_t):
        spans.setdefault(_frame(k) // _FPS, []).append(k)
    out = [(t, va + ks[0] * rows, va + (ks[-1] + 1) * rows) for t, ks in sorted(spans.items()) if sek_von <= t < sek_bis]
    if out and out[-1][2] > vb:
        raise ValueError("H3 Steady Lab: unexpected video layout")
    return out


def _schritt(opts):
    sig, sched = opts.get("sigmas"), opts.get("sample_sigmas")
    if sig is None or sched is None:
        return -1
    sched = sched.flatten().float().to(sig.device)
    return int(torch.argmin((sched - sig.flatten()[0].float()).abs()))


def _add(tok, delta, gen):
    if tok.dtype != torch.bfloat16:
        tok.add_(delta.to(tok.dtype))
        return
    bits = (tok.float() + delta).view(torch.int32)
    bits.add_(torch.randint(0, 1 << 16, bits.shape, device=bits.device, dtype=torch.int32, generator=gen))
    bits.bitwise_and_(-65536)
    tok.copy_(bits.view(torch.float32).to(torch.bfloat16))


def _patch(w, b, blk, staerken, sek, log):
    cache = {}

    @torch.no_grad()
    def block(args, extra):
        out = extra["original_block"](args)
        schritt = _schritt(args["transformer_options"])
        if staerken.get(schritt, 0.0) == 0.0:
            return out
        h, layout = out["img"], args["layout"]
        key = (layout.signature, h.device)
        if key not in cache:
            cache.clear()
            cache[key] = (_sekunden(layout, *sek), w[:, blk].to(h.device))
        spans, wb = cache[key]
        gen = torch.Generator(device=h.device)
        gen.manual_seed(1234 + 97 * schritt + (blk - 42) % 50)            # same as H3 Steady (indexed from block 42 there)
        st, schwelle = staerken[schritt], float(b[schritt, blk])
        korr = []
        for x in (h if h.dim() == 3 else [h]):
            for t, a, z in spans:
                tok = x[a:z]
                m = float((tok.float() @ wb[schritt]).mean())
                if m < schwelle:
                    _add(tok, st * (schwelle - m) * wb[schritt], gen)
                    korr.append(t)
        if log is not None:
            log(schritt, blk, korr, len(spans) * (h.shape[0] if h.dim() == 3 else 1), [t for t, _, _ in spans])
        return out

    return block


class H3SteadyLab(io.ComfyNode):
    @classmethod
    def define_schema(cls) -> io.Schema:
        return io.Schema(
            node_id="H3SteadyLab_Apply",
            display_name="H3 Steady Lab",
            category="advanced/model_patches",
            description="The stabilizer from H3 Steady with exposed controls (steps, blocks, threshold, time window). "
                        "Default settings = H3 Steady. Calibrated for er_sde, beta, 20 steps. Affects image tokens only.",
            inputs=[
                io.Model.Input("model"),
                io.Float.Input("staerke", display_name="Strength", default=2.0, min=-4.0, max=6.0, step=0.05,
                               tooltip="1.0 = up to the threshold, 2.0 = twice the distance. Negative = counter-test: "
                                       "pushes at-risk seconds further toward the failure side."),
                io.Float.Input("schritt_0", display_name="Step 0", default=0.0, min=0.0, max=3.0, step=0.05,
                               tooltip="Strength multiplier at step 0 (almost pure noise). 0 = off."),
                io.Float.Input("schritt_1", display_name="Step 1", default=1.0, min=0.0, max=3.0, step=0.05,
                               tooltip="Strength multiplier at step 1 (where H3 establishes the plan)."),
                io.Float.Input("schritt_2", display_name="Step 2", default=1.0, min=0.0, max=3.0, step=0.05,
                               tooltip="Strength multiplier at step 2."),
                io.Int.Input("block_von", display_name="Start block", default=42, min=0, max=49,
                             tooltip="First block. The direction mainly separated cases in blocks 32–48; blocks 0–25 and 49 do not separate them."),
                io.Int.Input("block_bis", display_name="End block", default=48, min=0, max=49,
                             tooltip="Last block (inclusive)."),
                io.Combo.Input("schwelle", display_name="Threshold", options=_SCHWELLEN, default=_SCHWELLEN[2],
                               tooltip="When a second is considered at risk: quantile of the 28 clean segments. "
                                       "Higher = more seconds are affected and pushed further."),
                io.Int.Input("sek_von", display_name="Start second", default=0, min=0, max=600,
                             tooltip="Only seconds from here onward (whole seconds of the generated clip, including context)."),
                io.Int.Input("sek_bis", display_name="End second", default=600, min=1, max=600,
                             tooltip="Only seconds before this point (exclusive). 3 to 5 = seconds 3.0 to 5.0."),
                io.Int.Input("offset", default=0, min=0, max=4096,
                             tooltip="Initial latent frames to leave untouched (chain from stage 2 onward: 27). 0 for regular clips."),
                io.Boolean.Input("protokoll", display_name="Logging", default=True,
                                 tooltip="Logs how many cells (second × block) were corrected at each step to the console."),
            ],
            outputs=[io.Model.Output()],
        )

    @classmethod
    def execute(cls, model, staerke, schritt_0, schritt_1, schritt_2, block_von, block_bis, schwelle, sek_von, sek_bis,
                offset, protokoll) -> io.NodeOutput:
        staerken = {s: staerke * f for s, f in enumerate((schritt_0, schritt_1, schritt_2)) if staerke * f != 0.0}
        bloecke = list(range(min(block_von, block_bis), max(block_von, block_bis) + 1))
        if not staerken or sek_bis <= sek_von:
            logging.info("H3 Steady Lab: nothing to do (strength, steps, or time window empty)")
            return io.NodeOutput(model)
        prof = load_file(_PROFIL)
        w, b = prof["w"], prof["b"][_SCHWELLEN.index(schwelle)]
        zaehler = {}

        def log(schritt, blk, korr, gesamt, sekunden):
            z = zaehler.setdefault(schritt, [0, 0, {}])
            z[0] += len(korr)
            z[1] += gesamt
            for t in korr:
                z[2][t] = z[2].get(t, 0) + 1
            if blk == bloecke[-1]:
                je = " ".join(f"{t}s:{z[2].get(t, 0)}" for t in sekunden)
                logging.info(f"H3 Steady Lab: step {schritt}: {z[0]} of {z[1]} cells corrected "
                             f"(blocks {bloecke[0]}–{bloecke[-1]}); per second, out of {len(bloecke)} blocks: {je}")
                zaehler.pop(schritt)

        m = model.clone()
        for blk in bloecke:
            m.set_model_patch_replace(_patch(w, b, blk, staerken, (offset, sek_von, sek_bis), log if protokoll else None),
                                      "dit", "double_block", blk)
        logging.info(f"H3 Steady Lab: strength {staerke}, steps {staerken}, blocks {bloecke[0]}–{bloecke[-1]}, "
                     f"threshold {schwelle}, seconds {sek_von}–{sek_bis}, offset {offset}")
        return io.NodeOutput(m)


class H3SteadyLabExtension(ComfyExtension):
    async def get_node_list(self) -> list[type[io.ComfyNode]]:
        return [H3SteadyLab]


async def comfy_entrypoint() -> H3SteadyLabExtension:
    return H3SteadyLabExtension()