someone-in-the-world's picture
Merge remote-tracking branch 'origin/develop' into add-mypy-strict-typecheck
c8ec0f7
Raw History Blame Contribute Delete
7.88 kB
"""WAMU_v3 image-to-video pipeline loading and sizing helpers.
Weight quantization (below) shrinks the full bf16 pipe from ~69GB, which made ZeroGPU's per-call
GPU init (loading the packed weights onto the GPU) exceed the requested call duration. The
transformers additionally get an AOT-compiled forward (FR-6) via `spaces.aoti_load()`, matching
the reference project: this is what makes `Float8DynamicActivationFloat8WeightConfig` (fp8x8
matmuls, both weights and activations quantized) safe to run. AOTInductor pre-plans a single
memory pool for every op in the compiled graph, including that config's per-layer
activation-quantize casts; run the same config eagerly (no AOT), as an earlier version of this
file did, and every one of those casts is an unplanned CUDA allocator request instead — that
churn reliably crashed ZeroGPU with an NVML assert (#35), fixed at the time by downgrading to
weight-only fp8 (no runtime activation quantization) as a stopgap. See #36 for the AOT
investigation (package-reuse feasibility) that unblocked switching back.
"""
from __future__ import annotations
import spaces
import torch
import torch._dynamo
from diffusers.pipelines.wan.pipeline_wan_i2v import WanImageToVideoPipeline
from PIL import Image
from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, Int8WeightOnlyConfig, quantize_
MODEL_ID = "thornmaze/WAMU_v3_WAN2.2_I2V_LIGHTNING"
# Precompiled AOT package for the transformer, from the reference project (zerogpu-aoti's
# wan2-2-fp8da-aoti-faster Space). Confirmed reusable for WAMU_v3 without recompiling (#36):
# WAMU_v3's transformer/transformer_2 share the exact same config (layer count, head dims, etc.)
# as the Wan2.2-I2V-A14B base model this package was compiled against — `spaces.aoti_load()`
# applies compiled kernels to an already-loaded model rather than embedding weights, so
# compatibility hinges on that shape/architecture match, not on WAMU_v3's fine-tuned weight
# values. "sm120" in the name matches ZeroGPU's current hardware (RTX Pro 6000 Blackwell).
AOTI_TRANSFORMER_REPO = "thornmaze/WanTransformer3DModel-sm120-cu130-raa"
MAX_DIM = 832
MIN_DIM = 480
SQUARE_DIM = 640
MULTIPLE_OF = 16
FIXED_FPS = 16
MIN_FRAMES_MODEL = 8
MAX_FRAMES_MODEL = 321
MIN_DURATION = round(MIN_FRAMES_MODEL / FIXED_FPS, 1)
MAX_DURATION = round(MAX_FRAMES_MODEL / FIXED_FPS, 1)
DEFAULT_PROMPT = "make this image come alive, cinematic motion, smooth animation"
DEFAULT_NEGATIVE_PROMPT = (
"色调艳丽, 过曝, 静态, 细节模糊不清, 字幕, 风格, 作品, 画作, 画面, 静止, 整体发灰, "
"最差质量, 低质量, JPEG压缩残留, 丑陋的, 残缺的, 多余的手指, 画得不好的手部, "
"画得不好的脸部, 畸形的, 毁容的, 形态畸形的肢体, 手指融合, 静止不动的画面, "
"杂乱的背景, 三条腿, 背景人很多, 倒着走"
)
def load_pipeline() -> WanImageToVideoPipeline:
pipe = WanImageToVideoPipeline.from_pretrained(MODEL_ID, torch_dtype=torch.bfloat16).to("cuda")
# Quantize to shrink the ~69GB bf16 pipe (drops per-call ZeroGPU init time and peak memory).
quantize_(pipe.text_encoder, Int8WeightOnlyConfig())
torch._dynamo.reset()
quantize_(pipe.transformer, Float8DynamicActivationFloat8WeightConfig())
torch._dynamo.reset()
quantize_(pipe.transformer_2, Float8DynamicActivationFloat8WeightConfig())
torch._dynamo.reset()
# AOT-compile the transformers (FR-6) — see module docstring and #36 for why this must come
# after the dynamic-activation quantize_() calls above, not before: the compiled graph is
# built from (and expects) that config's op layout, matching how the reference project's own
# usage example sequences it.
#
# Not yet validated against this repo's pinned torch==2.11.0: the package was compiled with
# torch==2.9.1+cu130, and AOTInductor artifacts can be ABI-sensitive across torch minor
# versions. Needs real-GPU testing on the dev Space — if loading fails here, the fallback is
# either pinning torch back to 2.9.1 or recompiling this package against 2.11.0.
spaces.aoti_load(module=pipe.transformer, repo_id=AOTI_TRANSFORMER_REPO)
spaces.aoti_load(module=pipe.transformer_2, repo_id=AOTI_TRANSFORMER_REPO)
return pipe
def resize_image(image: Image.Image) -> Image.Image:
"""Resize/crop to the model's supported dimension range, keeping a multiple-of-16 size."""
width, height = image.size
if width == height:
return image.resize((SQUARE_DIM, SQUARE_DIM), Image.Resampling.LANCZOS)
aspect_ratio = width / height
max_ar = MAX_DIM / MIN_DIM
min_ar = MIN_DIM / MAX_DIM
image_to_resize = image
if aspect_ratio > max_ar:
crop_width = int(round(height * max_ar))
left = (width - crop_width) // 2
image_to_resize = image.crop((left, 0, left + crop_width, height))
target_w, target_h = MAX_DIM, MIN_DIM
elif aspect_ratio < min_ar:
crop_height = int(round(width / min_ar))
top = (height - crop_height) // 2
image_to_resize = image.crop((0, top, width, top + crop_height))
target_w, target_h = MIN_DIM, MAX_DIM
else:
if width > height:
target_w = MAX_DIM
target_h = int(round(target_w / aspect_ratio))
else:
target_h = MAX_DIM
target_w = int(round(target_h * aspect_ratio))
final_w = max(MIN_DIM, min(MAX_DIM, round(target_w / MULTIPLE_OF) * MULTIPLE_OF))
final_h = max(MIN_DIM, min(MAX_DIM, round(target_h / MULTIPLE_OF) * MULTIPLE_OF))
return image_to_resize.resize((final_w, final_h), Image.Resampling.LANCZOS)
def resize_and_crop_to_match(target_image: Image.Image, reference_image: Image.Image) -> Image.Image:
"""Resize/crop `target_image` to exactly match `reference_image`'s size (the pipeline requires
the first and last frame to share dimensions)."""
ref_width, ref_height = reference_image.size
target_width, target_height = target_image.size
scale = max(ref_width / target_width, ref_height / target_height)
new_width, new_height = int(target_width * scale), int(target_height * scale)
resized = target_image.resize((new_width, new_height), Image.Resampling.LANCZOS)
left, top = (new_width - ref_width) // 2, (new_height - ref_height) // 2
return resized.crop((left, top, left + ref_width, top + ref_height))
def get_num_frames(duration_seconds: float) -> int:
raw = int(round(duration_seconds * FIXED_FPS))
raw = max(MIN_FRAMES_MODEL, min(MAX_FRAMES_MODEL, raw))
return ((raw - 1) // 4) * 4 + 1
def estimate_generation_seconds(
width: int, height: int, num_frames: int, steps: int, guidance_scale: float
) -> float:
"""Rough @spaces.GPU duration budget, calibrated from the reference wan2-2-i2v-v3 Space.
Exact limits are still an open item (SRS Section 4, #3) pending empirical testing on the
dev Space; this heuristic is a starting point, not a final calibration.
Kept conservative even now that the transformers are AOT-compiled (#36): the reference
project's numbers below assume a validated AOT setup, and ours hasn't been measured on real
GPU yet (see load_pipeline's AOT caveat) — recalibrate once that's confirmed working.
"""
base_frames_height_width = 81 * 832 * 624
base_step_duration = 5.0
factor = num_frames * width * height / base_frames_height_width
step_duration = base_step_duration * float(factor**1.5)
gen_time = steps * step_duration
if guidance_scale > 1:
gen_time *= 2.4
# Padded well above the reference project's flat overhead (its pipe loads a validated AOT
# setup already proven on its own GPU quota) until ours is confirmed working and measured
# empirically on the dev Space (#36).
init_overhead = 60
return init_overhead + gen_time